Merge branch 'dev' of https://github.com/SyncrowIOT/backend into SP-1315-BE-API-rate-limit

This commit is contained in:
hannathkadher
2025-04-24 12:21:49 +04:00
11 changed files with 294 additions and 59 deletions

View File

@ -0,0 +1,8 @@
import { AsyncLocalStorage } from 'async_hooks';
export interface RequestContextStore {
requestId?: string;
userId?: string;
}
export const requestContext = new AsyncLocalStorage<RequestContextStore>();

View File

@ -43,19 +43,17 @@ import { SubspaceProductAllocationEntity } from '../modules/space/entities/subsp
import { SubspaceEntity } from '../modules/space/entities/subspace/subspace.entity';
import { TagEntity } from '../modules/space/entities/tag.entity';
import { ClientEntity } from '../modules/client/entities';
import { TypeOrmWinstonLogger } from '@app/common/logger/services/typeorm.logger';
import { createLogger } from 'winston';
import { winstonLoggerOptions } from 'src/common/filters/http-exception/logger/winston.logger';
import { winstonLoggerOptions } from '../logger/services/winston.logger';
@Module({
imports: [
TypeOrmModule.forRootAsync({
imports: [ConfigModule],
inject: [ConfigService],
useFactory: (configService: ConfigService) => {
const sslEnabled = JSON.parse(configService.get<string>('DB_SSL'));
const winstonLogger = createLogger(winstonLoggerOptions);
const typeOrmLogger = new TypeOrmWinstonLogger(winstonLogger);
return {
name: 'default',
type: 'postgres',
@ -64,20 +62,6 @@ import { winstonLoggerOptions } from 'src/common/filters/http-exception/logger/w
username: configService.get('DB_USER'),
password: configService.get('DB_PASSWORD'),
database: configService.get('DB_NAME'),
synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))),
logging: ['query', 'error', 'warn', 'schema', 'migration'],
namingStrategy: new SnakeNamingStrategy(),
ssl: sslEnabled ? { rejectUnauthorized: false } : false,
extra: {
charset: 'utf8mb4',
max: 20,
idleTimeoutMillis: 5000,
connectionTimeoutMillis: 11000,
maxUses: 7500,
...(sslEnabled && {
ssl: { rejectUnauthorized: false }, // Required for Azure PostgreSQL
}),
},
entities: [
NewTagEntity,
ProjectEntity,
@ -94,6 +78,7 @@ import { winstonLoggerOptions } from 'src/common/filters/http-exception/logger/w
SubspaceEntity,
TagEntity,
UserSpaceEntity,
DeviceUserPermissionEntity,
RoleTypeEntity,
UserNotificationEntity,
DeviceNotificationEntity,
@ -117,6 +102,20 @@ import { winstonLoggerOptions } from 'src/common/filters/http-exception/logger/w
SubspaceProductAllocationEntity,
ClientEntity,
],
namingStrategy: new SnakeNamingStrategy(),
synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))),
logging: ['query', 'error', 'warn', 'schema', 'migration'],
logger: typeOrmLogger,
extra: {
charset: 'utf8mb4',
max: 20, // set pool max size
idleTimeoutMillis: 5000, // close idle clients after 5 second
connectionTimeoutMillis: 11_000, // return an error after 11 second if connection could not be established
maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion)
},
continuationLocalStorage: true,
ssl: Boolean(JSON.parse(configService.get('DB_SSL'))),
};
},
}),

View File

@ -0,0 +1,12 @@
// src/common/logger/logger.module.ts
import { Module } from '@nestjs/common';
import { WinstonModule } from 'nest-winston';
import { winstonLoggerOptions } from './services/winston.logger';
import { TypeOrmWinstonLogger } from './services/typeorm.logger';
@Module({
imports: [WinstonModule.forRoot(winstonLoggerOptions)],
providers: [TypeOrmWinstonLogger],
exports: [TypeOrmWinstonLogger],
})
export class LoggerModule {}

View File

@ -0,0 +1,72 @@
import { Logger as WinstonLogger } from 'winston';
import { Logger as TypeOrmLogger } from 'typeorm';
import { requestContext } from '@app/common/context/request-context';
const ERROR_THRESHOLD = 2000;
export class TypeOrmWinstonLogger implements TypeOrmLogger {
constructor(private readonly logger: WinstonLogger) {}
private getContext() {
const context = requestContext.getStore();
return {
requestId: context?.requestId ?? 'N/A',
userId: context?.userId ?? null,
};
}
private extractTable(query: string): string {
const match =
query.match(/from\s+["`]?(\w+)["`]?/i) ||
query.match(/into\s+["`]?(\w+)["`]?/i);
return match?.[1] ?? 'unknown';
}
logQuery(query: string, parameters?: any[]) {
const context = this.getContext();
this.logger.debug(`[DB][QUERY] ${query}`, {
...context,
table: this.extractTable(query),
parameters,
});
}
logQueryError(error: string | Error, query: string, parameters?: any[]) {
const context = this.getContext();
this.logger.error(`[DB][ERROR] ${query}`, {
...context,
table: this.extractTable(query),
parameters,
error,
});
}
logQuerySlow(time: number, query: string, parameters?: any[]) {
const context = this.getContext();
const severity = time > ERROR_THRESHOLD ? 'error' : 'warn';
const label = severity === 'error' ? 'VERY SLOW' : 'SLOW';
this.logger[severity](`[DB][${label} > ${time}ms] ${query}`, {
...context,
table: this.extractTable(query),
parameters,
duration: `${time}ms`,
severity,
});
}
logSchemaBuild(message: string) {
this.logger.info(`[DB][SCHEMA] ${message}`);
}
logMigration(message: string) {
this.logger.info(`[DB][MIGRATION] ${message}`);
}
log(level: 'log' | 'info' | 'warn', message: any) {
this.logger.log({
level,
message: `[DB] ${message}`,
});
}
}

View File

@ -2,6 +2,8 @@ import { utilities as nestWinstonModuleUtilities } from 'nest-winston';
import * as winston from 'winston';
export const winstonLoggerOptions: winston.LoggerOptions = {
level:
process.env.AZURE_POSTGRESQL_DATABASE === 'development' ? 'debug' : 'error',
transports: [
new winston.transports.Console({
format: winston.format.combine(

View File

@ -0,0 +1,14 @@
import { Injectable, NestMiddleware } from '@nestjs/common';
import { requestContext } from '../context/request-context';
import { v4 as uuidv4 } from 'uuid';
@Injectable()
export class RequestContextMiddleware implements NestMiddleware {
use(req: any, res: any, next: () => void) {
const context = {
requestId: req.headers['x-request-id'] || uuidv4(),
};
requestContext.run(context, () => next());
}
}

View File

@ -1,25 +1,15 @@
WITH params AS (
SELECT
NULL::uuid AS device_id, -- filter: specific device (or NULL for all)
NULL::date AS input_date, -- filter: start date (or NULL for open range) -- filter: end date (or NULL for open range)
NULL::int AS hour, -- filter: hour of day (or NULL for all hours)
NULL::int AS min_kw -- filter: min kW consumed (or NULL for no filter)
),
total_energy AS (
WITH total_energy AS (
SELECT
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log, params
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumed'
AND (params.device_id IS NULL OR log.device_id = params.device_id)
AND (params.input_date IS NULL OR log.event_time::date >= params.input_date)
AND (params.input_date IS NULL OR log.event_time::date <= params.input_date)
AND (params.hour IS NULL OR EXTRACT(HOUR FROM log.event_time) = params.hour)
GROUP BY log.device_id, log.event_time::date, EXTRACT(HOUR FROM log.event_time)
GROUP BY 1,2,3,4,5
),
energy_phase_A AS (
@ -27,15 +17,13 @@ energy_phase_A AS (
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log, params
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumedA'
AND (params.device_id IS NULL OR log.device_id = params.device_id)
AND (params.input_date IS NULL OR log.event_time::date >= params.input_date)
AND (params.input_date IS NULL OR log.event_time::date <= params.input_date)
AND (params.hour IS NULL OR EXTRACT(HOUR FROM log.event_time) = params.hour)
GROUP BY log.device_id, log.event_time::date, EXTRACT(HOUR FROM log.event_time)
GROUP BY 1,2,3,4,5
),
energy_phase_B AS (
@ -43,15 +31,13 @@ energy_phase_B AS (
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log, params
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumedB'
AND (params.device_id IS NULL OR log.device_id = params.device_id)
AND (params.input_date IS NULL OR log.event_time::date >= params.input_date)
AND (params.input_date IS NULL OR log.event_time::date <= params.input_date)
AND (params.hour IS NULL OR EXTRACT(HOUR FROM log.event_time) = params.hour)
GROUP BY log.device_id, log.event_time::date, EXTRACT(HOUR FROM log.event_time)
GROUP BY 1,2,3,4,5
),
energy_phase_C AS (
@ -59,20 +45,20 @@ energy_phase_C AS (
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log, params
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumedC'
AND (params.device_id IS NULL OR log.device_id = params.device_id)
AND (params.input_date IS NULL OR log.event_time::date >= params.input_date)
AND (params.input_date IS NULL OR log.event_time::date <= params.input_date)
AND (params.hour IS NULL OR EXTRACT(HOUR FROM log.event_time) = params.hour)
GROUP BY log.device_id, log.event_time::date, EXTRACT(HOUR FROM log.event_time)
GROUP BY 1,2,3,4,5
)
SELECT
t.device_id,
t.date,
t.event_year::text,
t.event_month,
t.hour,
(t.max_value - t.min_value) AS energy_consumed_kW,
(a.max_value - a.min_value) AS energy_consumed_A,
@ -82,6 +68,4 @@ FROM total_energy t
JOIN energy_phase_A a ON t.device_id = a.device_id AND t.date = a.date AND t.hour = a.hour
JOIN energy_phase_B b ON t.device_id = b.device_id AND t.date = b.date AND t.hour = b.hour
JOIN energy_phase_C c ON t.device_id = c.device_id AND t.date = c.date AND t.hour = c.hour
JOIN params p ON TRUE
WHERE (p.min_kw IS NULL OR (t.max_value - t.min_value) >= p.min_kw)
ORDER BY t.device_id, t.date, t.hour;
ORDER BY 1,2;

View File

@ -0,0 +1,135 @@
WITH total_energy AS (
SELECT
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumed'
GROUP BY 1,2,3,4,5
),
energy_phase_A AS (
SELECT
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumedA'
GROUP BY 1,2,3,4,5
),
energy_phase_B AS (
SELECT
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumedB'
GROUP BY 1,2,3,4,5
),
energy_phase_C AS (
SELECT
log.device_id,
log.event_time::date AS date,
EXTRACT(HOUR FROM log.event_time) AS hour,
TO_CHAR(log.event_time, 'MM-YYYY') AS event_month,
EXTRACT(YEAR FROM log.event_time)::int AS event_year,
MIN(log.value)::integer AS min_value,
MAX(log.value)::integer AS max_value
FROM "device-status-log" log
WHERE log.code = 'EnergyConsumedC'
GROUP BY 1,2,3,4,5
)
, final_data as (
SELECT
t.device_id,
t.date,
t.event_year::text,
t.event_month,
t.hour,
(t.max_value - t.min_value) AS energy_consumed_kW,
(a.max_value - a.min_value) AS energy_consumed_A,
(b.max_value - b.min_value) AS energy_consumed_B,
(c.max_value - c.min_value) AS energy_consumed_C
FROM total_energy t
JOIN energy_phase_A a ON t.device_id = a.device_id AND t.date = a.date AND t.hour = a.hour
JOIN energy_phase_B b ON t.device_id = b.device_id AND t.date = b.date AND t.hour = b.hour
JOIN energy_phase_C c ON t.device_id = c.device_id AND t.date = c.date AND t.hour = c.hour
ORDER BY 1,2)
INSERT INTO public."power-clamp-energy-consumed-daily"(
device_uuid,
energy_consumed_kw,
energy_consumed_a,
energy_consumed_b,
energy_consumed_c,
date
)
SELECT
device_id,
SUM(CAST(energy_consumed_kw AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_a AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_b AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_c AS NUMERIC))::VARCHAR,
date
FROM final_data
GROUP BY device_id, date;
INSERT INTO public."power-clamp-energy-consumed-hourly"(
device_uuid,
energy_consumed_kw,
energy_consumed_a,
energy_consumed_b,
energy_consumed_c,
date,
hour
)
SELECT
device_id,
SUM(CAST(energy_consumed_kw AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_a AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_b AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_c AS NUMERIC))::VARCHAR,
date,
hour
FROM final_data
GROUP BY 1,6,7
INSERT INTO public."power-clamp-energy-consumed-monthly"(
device_uuid,
energy_consumed_kw,
energy_consumed_a,
energy_consumed_b,
energy_consumed_c,
month
)
SELECT
device_id,
SUM(CAST(energy_consumed_kw AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_a AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_b AS NUMERIC))::VARCHAR,
SUM(CAST(energy_consumed_c AS NUMERIC))::VARCHAR,
TO_CHAR(date, 'MM-YYYY')
FROM final_data
GROUP BY 1,6;

View File

@ -33,10 +33,10 @@ import { ClientModule } from './client/client.module';
import { DeviceCommissionModule } from './commission-device/commission-device.module';
import { PowerClampModule } from './power-clamp/power-clamp.module';
import { WinstonModule } from 'nest-winston';
import { winstonLoggerOptions } from './common/filters/http-exception/logger/winston.logger';
import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler';
import { HealthModule } from './health/health.module';
import { winstonLoggerOptions } from '../libs/common/src/logger/services/winston.logger';
@Module({
imports: [
ConfigModule.forRoot({

View File

@ -679,12 +679,14 @@ export class DeviceService {
},
});
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { productId, id, uuid, name, ...rest } = camelCaseResponse.result;
const { productId, id, productName, uuid, name, ...rest } =
camelCaseResponse.result;
return {
...rest,
productUuid: product.uuid,
name: deviceDetails.name,
productName: product.name,
} as GetDeviceDetailsInterface;
} catch (error) {
console.log('error', error);

View File

@ -7,6 +7,9 @@ import { ValidationPipe } from '@nestjs/common';
import { json, urlencoded } from 'body-parser';
import { SeederService } from '@app/common/seed/services/seeder.service';
import { HttpExceptionFilter } from './common/filters/http-exception/http-exception.filter';
import { Logger } from '@nestjs/common';
import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston';
import { RequestContextMiddleware } from '@app/common/middleware/request-context.middleware';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
@ -18,6 +21,8 @@ async function bootstrap() {
app.use(urlencoded({ limit: '1mb', extended: true }));
app.useGlobalFilters(new HttpExceptionFilter());
app.use(new RequestContextMiddleware().use);
app.use(
rateLimit({
windowMs: 5 * 60 * 1000,
@ -43,14 +48,16 @@ async function bootstrap() {
);
const seederService = app.get(SeederService);
const logger = app.get<Logger>(WINSTON_MODULE_NEST_PROVIDER);
try {
await seederService.seed();
console.log('Seeding complete!');
logger.log('Seeding complete!');
} catch (error) {
console.error('Seeding failed!', error);
logger.error('Seeding failed!', error.stack || error);
}
console.log('Starting auth at port ...', process.env.PORT || 4000);
logger.log('Starting auth at port ...', process.env.PORT || 4000);
await app.listen(process.env.PORT || 4000);
}
bootstrap();