From fcb27155d882119a2bca6c107283f22f98795f06 Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Tue, 22 Apr 2025 10:20:48 +0400 Subject: [PATCH 1/9] typeorm logger --- libs/common/src/context/request-context.ts | 8 ++ libs/common/src/database/database.module.ts | 131 ++++++++++-------- libs/common/src/logger/logger.module.ts | 12 ++ .../src/logger/services/typeorm.logger.ts | 64 +++++++++ .../src/logger/services}/winston.logger.ts | 1 + .../middleware/request-context.middleware.ts | 14 ++ src/app.module.ts | 2 +- src/main.ts | 13 +- 8 files changed, 180 insertions(+), 65 deletions(-) create mode 100644 libs/common/src/context/request-context.ts create mode 100644 libs/common/src/logger/logger.module.ts create mode 100644 libs/common/src/logger/services/typeorm.logger.ts rename {src/common/filters/http-exception/logger => libs/common/src/logger/services}/winston.logger.ts (97%) create mode 100644 libs/common/src/middleware/request-context.middleware.ts diff --git a/libs/common/src/context/request-context.ts b/libs/common/src/context/request-context.ts new file mode 100644 index 0000000..7067bfc --- /dev/null +++ b/libs/common/src/context/request-context.ts @@ -0,0 +1,8 @@ +import { AsyncLocalStorage } from 'async_hooks'; + +export interface RequestContextStore { + requestId?: string; + userId?: string; +} + +export const requestContext = new AsyncLocalStorage(); diff --git a/libs/common/src/database/database.module.ts b/libs/common/src/database/database.module.ts index a25cf35..7ebec3b 100644 --- a/libs/common/src/database/database.module.ts +++ b/libs/common/src/database/database.module.ts @@ -43,73 +43,82 @@ import { SubspaceProductAllocationEntity } from '../modules/space/entities/subsp import { SubspaceEntity } from '../modules/space/entities/subspace/subspace.entity'; import { TagEntity } from '../modules/space/entities/tag.entity'; import { ClientEntity } from '../modules/client/entities'; +import { TypeOrmWinstonLogger } from '@app/common/logger/services/typeorm.logger'; +import { createLogger } from 'winston'; +import { winstonLoggerOptions } from '../logger/services/winston.logger'; @Module({ imports: [ TypeOrmModule.forRootAsync({ imports: [ConfigModule], inject: [ConfigService], - useFactory: (configService: ConfigService) => ({ - name: 'default', - type: 'postgres', - host: configService.get('DB_HOST'), - port: configService.get('DB_PORT'), - username: configService.get('DB_USER'), - password: configService.get('DB_PASSWORD'), - database: configService.get('DB_NAME'), - entities: [ - NewTagEntity, - ProjectEntity, - UserEntity, - UserSessionEntity, - UserOtpEntity, - ProductEntity, - DeviceUserPermissionEntity, - DeviceEntity, - PermissionTypeEntity, - CommunityEntity, - SpaceEntity, - SpaceLinkEntity, - SubspaceEntity, - TagEntity, - UserSpaceEntity, - DeviceUserPermissionEntity, - RoleTypeEntity, - UserNotificationEntity, - DeviceNotificationEntity, - RegionEntity, - TimeZoneEntity, - VisitorPasswordEntity, - DeviceStatusLogEntity, - SceneEntity, - SceneIconEntity, - SceneDeviceEntity, - SpaceModelEntity, - SubspaceModelEntity, - TagModel, - InviteUserEntity, - InviteUserSpaceEntity, - InviteSpaceEntity, - AutomationEntity, - SpaceModelProductAllocationEntity, - SubspaceModelProductAllocationEntity, - SpaceProductAllocationEntity, - SubspaceProductAllocationEntity, - ClientEntity, - ], - namingStrategy: new SnakeNamingStrategy(), - synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))), - logging: false, - extra: { - charset: 'utf8mb4', - max: 20, // set pool max size - idleTimeoutMillis: 5000, // close idle clients after 5 second - connectionTimeoutMillis: 11_000, // return an error after 11 second if connection could not be established - maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion) - }, - continuationLocalStorage: true, - ssl: Boolean(JSON.parse(configService.get('DB_SSL'))), - }), + useFactory: (configService: ConfigService) => { + const winstonLogger = createLogger(winstonLoggerOptions); + const typeOrmLogger = new TypeOrmWinstonLogger(winstonLogger); + return { + name: 'default', + type: 'postgres', + host: configService.get('DB_HOST'), + port: configService.get('DB_PORT'), + username: configService.get('DB_USER'), + password: configService.get('DB_PASSWORD'), + database: configService.get('DB_NAME'), + entities: [ + NewTagEntity, + ProjectEntity, + UserEntity, + UserSessionEntity, + UserOtpEntity, + ProductEntity, + DeviceUserPermissionEntity, + DeviceEntity, + PermissionTypeEntity, + CommunityEntity, + SpaceEntity, + SpaceLinkEntity, + SubspaceEntity, + TagEntity, + UserSpaceEntity, + DeviceUserPermissionEntity, + RoleTypeEntity, + UserNotificationEntity, + DeviceNotificationEntity, + RegionEntity, + TimeZoneEntity, + VisitorPasswordEntity, + DeviceStatusLogEntity, + SceneEntity, + SceneIconEntity, + SceneDeviceEntity, + SpaceModelEntity, + SubspaceModelEntity, + TagModel, + InviteUserEntity, + InviteUserSpaceEntity, + InviteSpaceEntity, + AutomationEntity, + SpaceModelProductAllocationEntity, + SubspaceModelProductAllocationEntity, + SpaceProductAllocationEntity, + SubspaceProductAllocationEntity, + ClientEntity, + ], + namingStrategy: new SnakeNamingStrategy(), + synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))), + logging: true, + logger: typeOrmLogger, + extra: { + charset: 'utf8mb4', + max: 20, // set pool max size + idleTimeoutMillis: 5000, // close idle clients after 5 second + connectionTimeoutMillis: 11_000, // return an error after 11 second if connection could not be established + maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion) + }, + continuationLocalStorage: true, + ssl: Boolean(JSON.parse(configService.get('DB_SSL'))), + }; + }, }), ], + providers: [TypeOrmWinstonLogger], }) export class DatabaseModule {} diff --git a/libs/common/src/logger/logger.module.ts b/libs/common/src/logger/logger.module.ts new file mode 100644 index 0000000..50b241a --- /dev/null +++ b/libs/common/src/logger/logger.module.ts @@ -0,0 +1,12 @@ +// src/common/logger/logger.module.ts +import { Module } from '@nestjs/common'; +import { WinstonModule } from 'nest-winston'; +import { winstonLoggerOptions } from './services/winston.logger'; +import { TypeOrmWinstonLogger } from './services/typeorm.logger'; + +@Module({ + imports: [WinstonModule.forRoot(winstonLoggerOptions)], + providers: [TypeOrmWinstonLogger], + exports: [TypeOrmWinstonLogger], +}) +export class LoggerModule {} diff --git a/libs/common/src/logger/services/typeorm.logger.ts b/libs/common/src/logger/services/typeorm.logger.ts new file mode 100644 index 0000000..098959b --- /dev/null +++ b/libs/common/src/logger/services/typeorm.logger.ts @@ -0,0 +1,64 @@ +import { Logger as WinstonLogger } from 'winston'; +import { Logger as TypeOrmLogger } from 'typeorm'; +import { requestContext } from '@app/common/context/request-context'; + +export class TypeOrmWinstonLogger implements TypeOrmLogger { + constructor( + private readonly logger: WinstonLogger, + private readonly slowQueryThreshold = 500, // ms + ) {} + + logQuery(query: string, parameters?: any[]) { + const context = requestContext.getStore(); + const requestId = context?.requestId ?? 'N/A'; + const start = Date.now(); + + const timeout = setTimeout(() => { + const duration = Date.now() - start; + + const isSlow = duration > this.slowQueryThreshold; + this.logger[isSlow ? 'warn' : 'debug'](`[DB][QUERY] ${query}`, { + requestId, + parameters, + duration: `${duration}ms`, + isSlow, + }); + }, 0); + + // Just ensures the setTimeout fires after this function returns + clearTimeout(timeout); + } + + logQueryError(error: string | Error, query: string, parameters?: any[]) { + const requestId = requestContext.getStore()?.requestId ?? 'N/A'; + this.logger.error(`[DB][ERROR] ${query}`, { + requestId, + parameters, + error, + }); + } + + logQuerySlow(time: number, query: string, parameters?: any[]) { + const requestId = requestContext.getStore()?.requestId ?? 'N/A'; + this.logger.warn(`🔥 [DB][SLOW > ${time}ms] ${query}`, { + requestId, + parameters, + time, + }); + } + + logSchemaBuild(message: string) { + this.logger.info(`[DB][SCHEMA] ${message}`); + } + + logMigration(message: string) { + this.logger.info(`[DB][MIGRATION] ${message}`); + } + + log(level: 'log' | 'info' | 'warn', message: any) { + this.logger.log({ + level, + message: `[DB] ${message}`, + }); + } +} diff --git a/src/common/filters/http-exception/logger/winston.logger.ts b/libs/common/src/logger/services/winston.logger.ts similarity index 97% rename from src/common/filters/http-exception/logger/winston.logger.ts rename to libs/common/src/logger/services/winston.logger.ts index 2c024ef..7f0e806 100644 --- a/src/common/filters/http-exception/logger/winston.logger.ts +++ b/libs/common/src/logger/services/winston.logger.ts @@ -2,6 +2,7 @@ import { utilities as nestWinstonModuleUtilities } from 'nest-winston'; import * as winston from 'winston'; export const winstonLoggerOptions: winston.LoggerOptions = { + level: 'debug', transports: [ new winston.transports.Console({ format: winston.format.combine( diff --git a/libs/common/src/middleware/request-context.middleware.ts b/libs/common/src/middleware/request-context.middleware.ts new file mode 100644 index 0000000..1560d7a --- /dev/null +++ b/libs/common/src/middleware/request-context.middleware.ts @@ -0,0 +1,14 @@ +import { Injectable, NestMiddleware } from '@nestjs/common'; +import { requestContext } from '../context/request-context'; +import { v4 as uuidv4 } from 'uuid'; + +@Injectable() +export class RequestContextMiddleware implements NestMiddleware { + use(req: any, res: any, next: () => void) { + const context = { + requestId: req.headers['x-request-id'] || uuidv4(), + }; + + requestContext.run(context, () => next()); + } +} diff --git a/src/app.module.ts b/src/app.module.ts index f07ff20..1fc6c2d 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -33,7 +33,7 @@ import { ClientModule } from './client/client.module'; import { DeviceCommissionModule } from './commission-device/commission-device.module'; import { PowerClampModule } from './power-clamp/power-clamp.module'; import { WinstonModule } from 'nest-winston'; -import { winstonLoggerOptions } from './common/filters/http-exception/logger/winston.logger'; +import { winstonLoggerOptions } from '../libs/common/src/logger/services/winston.logger'; @Module({ imports: [ ConfigModule.forRoot({ diff --git a/src/main.ts b/src/main.ts index 6dbfc45..d337a66 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,6 +7,9 @@ import { ValidationPipe } from '@nestjs/common'; import { json, urlencoded } from 'body-parser'; import { SeederService } from '@app/common/seed/services/seeder.service'; import { HttpExceptionFilter } from './common/filters/http-exception/http-exception.filter'; +import { Logger } from '@nestjs/common'; +import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; +import { RequestContextMiddleware } from '@app/common/middleware/request-context.middleware'; async function bootstrap() { const app = await NestFactory.create(AppModule); @@ -18,6 +21,8 @@ async function bootstrap() { app.use(urlencoded({ limit: '1mb', extended: true })); app.useGlobalFilters(new HttpExceptionFilter()); + app.use(new RequestContextMiddleware().use); + app.use( rateLimit({ windowMs: 5 * 60 * 1000, @@ -43,14 +48,16 @@ async function bootstrap() { ); const seederService = app.get(SeederService); + const logger = app.get(WINSTON_MODULE_NEST_PROVIDER); + try { await seederService.seed(); - console.log('Seeding complete!'); + logger.log('Seeding complete!'); } catch (error) { - console.error('Seeding failed!', error); + logger.error('Seeding failed!', error.stack || error); } - console.log('Starting auth at port ...', process.env.PORT || 4000); + logger.log('Starting auth at port ...', process.env.PORT || 4000); await app.listen(process.env.PORT || 4000); } bootstrap(); From cff947a84afbf5add07cdc753528104a3cf807e1 Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Tue, 22 Apr 2025 21:44:46 +0400 Subject: [PATCH 2/9] database config --- libs/common/src/database/database.module.ts | 3 +- .../src/logger/services/typeorm.logger.ts | 64 +++++++++++-------- .../src/logger/services/winston.logger.ts | 3 +- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/libs/common/src/database/database.module.ts b/libs/common/src/database/database.module.ts index 7ebec3b..8540d22 100644 --- a/libs/common/src/database/database.module.ts +++ b/libs/common/src/database/database.module.ts @@ -104,7 +104,8 @@ import { winstonLoggerOptions } from '../logger/services/winston.logger'; ], namingStrategy: new SnakeNamingStrategy(), synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))), - logging: true, + logging: ['query', 'error', 'warn', 'schema', 'migration'], + logger: typeOrmLogger, extra: { charset: 'utf8mb4', diff --git a/libs/common/src/logger/services/typeorm.logger.ts b/libs/common/src/logger/services/typeorm.logger.ts index 098959b..bd5a14a 100644 --- a/libs/common/src/logger/services/typeorm.logger.ts +++ b/libs/common/src/logger/services/typeorm.logger.ts @@ -2,48 +2,56 @@ import { Logger as WinstonLogger } from 'winston'; import { Logger as TypeOrmLogger } from 'typeorm'; import { requestContext } from '@app/common/context/request-context'; +const ERROR_THRESHOLD = 2000; + export class TypeOrmWinstonLogger implements TypeOrmLogger { - constructor( - private readonly logger: WinstonLogger, - private readonly slowQueryThreshold = 500, // ms - ) {} + constructor(private readonly logger: WinstonLogger) {} + + private getContext() { + const context = requestContext.getStore(); + return { + requestId: context?.requestId ?? 'N/A', + userId: context?.userId ?? null, + }; + } + + private extractTable(query: string): string { + const match = + query.match(/from\s+["`]?(\w+)["`]?/i) || + query.match(/into\s+["`]?(\w+)["`]?/i); + return match?.[1] ?? 'unknown'; + } logQuery(query: string, parameters?: any[]) { - const context = requestContext.getStore(); - const requestId = context?.requestId ?? 'N/A'; - const start = Date.now(); - - const timeout = setTimeout(() => { - const duration = Date.now() - start; - - const isSlow = duration > this.slowQueryThreshold; - this.logger[isSlow ? 'warn' : 'debug'](`[DB][QUERY] ${query}`, { - requestId, - parameters, - duration: `${duration}ms`, - isSlow, - }); - }, 0); - - // Just ensures the setTimeout fires after this function returns - clearTimeout(timeout); + const context = this.getContext(); + this.logger.debug(`[DB][QUERY] ${query}`, { + ...context, + table: this.extractTable(query), + parameters, + }); } logQueryError(error: string | Error, query: string, parameters?: any[]) { - const requestId = requestContext.getStore()?.requestId ?? 'N/A'; + const context = this.getContext(); this.logger.error(`[DB][ERROR] ${query}`, { - requestId, + ...context, + table: this.extractTable(query), parameters, error, }); } logQuerySlow(time: number, query: string, parameters?: any[]) { - const requestId = requestContext.getStore()?.requestId ?? 'N/A'; - this.logger.warn(`🔥 [DB][SLOW > ${time}ms] ${query}`, { - requestId, + const context = this.getContext(); + const severity = time > ERROR_THRESHOLD ? 'error' : 'warn'; + const label = severity === 'error' ? 'VERY SLOW' : 'SLOW'; + + this.logger[severity](`[DB][${label} > ${time}ms] ${query}`, { + ...context, + table: this.extractTable(query), parameters, - time, + duration: `${time}ms`, + severity, }); } diff --git a/libs/common/src/logger/services/winston.logger.ts b/libs/common/src/logger/services/winston.logger.ts index 7f0e806..34b6a75 100644 --- a/libs/common/src/logger/services/winston.logger.ts +++ b/libs/common/src/logger/services/winston.logger.ts @@ -2,7 +2,8 @@ import { utilities as nestWinstonModuleUtilities } from 'nest-winston'; import * as winston from 'winston'; export const winstonLoggerOptions: winston.LoggerOptions = { - level: 'debug', + level: + process.env.AZURE_POSTGRESQL_DATABASE === 'development' ? 'debug' : 'error', transports: [ new winston.transports.Console({ format: winston.format.combine( From cb2056d1b3820915e75133213cd09b999963be80 Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Wed, 23 Apr 2025 10:56:03 +0400 Subject: [PATCH 3/9] added health check --- package-lock.json | 189 +++++++++++++++++++- package.json | 3 + src/app.module.ts | 13 +- src/health/controllers/health.controller.ts | 35 ++++ src/health/controllers/index.ts | 1 + src/health/health.module.ts | 11 ++ 6 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 src/health/controllers/health.controller.ts create mode 100644 src/health/controllers/index.ts create mode 100644 src/health/health.module.ts diff --git a/package-lock.json b/package-lock.json index 9f26111..eaf972a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@fast-csv/format": "^5.0.2", + "@nestjs/axios": "^4.0.0", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.2.0", "@nestjs/core": "^10.0.0", @@ -18,6 +19,8 @@ "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.0.0", "@nestjs/swagger": "^7.3.0", + "@nestjs/terminus": "^11.0.0", + "@nestjs/throttler": "^6.4.0", "@nestjs/typeorm": "^10.0.2", "@nestjs/websockets": "^10.3.8", "@tuya/tuya-connector-nodejs": "^2.1.2", @@ -2232,6 +2235,17 @@ "integrity": "sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==", "license": "MIT" }, + "node_modules/@nestjs/axios": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-4.0.0.tgz", + "integrity": "sha512-1cB+Jyltu/uUPNQrpUimRHEQHrnQrpLzVj6dU3dgn6iDDDdahr10TgHFGTmw5VuJ9GzKZsCLDL78VSwJAs/9JQ==", + "license": "MIT", + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "axios": "^1.3.1", + "rxjs": "^7.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "10.4.9", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.4.9.tgz", @@ -2581,6 +2595,76 @@ } } }, + "node_modules/@nestjs/terminus": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/@nestjs/terminus/-/terminus-11.0.0.tgz", + "integrity": "sha512-c55LOo9YGovmQHtFUMa/vDaxGZ2cglMTZejqgHREaApt/GArTfgYYGwhRXPLq8ZwiQQlLuYB+79e9iA8mlDSLA==", + "license": "MIT", + "dependencies": { + "boxen": "5.1.2", + "check-disk-space": "3.4.0" + }, + "peerDependencies": { + "@grpc/grpc-js": "*", + "@grpc/proto-loader": "*", + "@mikro-orm/core": "*", + "@mikro-orm/nestjs": "*", + "@nestjs/axios": "^2.0.0 || ^3.0.0 || ^4.0.0", + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0", + "@nestjs/microservices": "^10.0.0 || ^11.0.0", + "@nestjs/mongoose": "^11.0.0", + "@nestjs/sequelize": "^10.0.0 || ^11.0.0", + "@nestjs/typeorm": "^10.0.0 || ^11.0.0", + "@prisma/client": "*", + "mongoose": "*", + "reflect-metadata": "0.1.x || 0.2.x", + "rxjs": "7.x", + "sequelize": "*", + "typeorm": "*" + }, + "peerDependenciesMeta": { + "@grpc/grpc-js": { + "optional": true + }, + "@grpc/proto-loader": { + "optional": true + }, + "@mikro-orm/core": { + "optional": true + }, + "@mikro-orm/nestjs": { + "optional": true + }, + "@nestjs/axios": { + "optional": true + }, + "@nestjs/microservices": { + "optional": true + }, + "@nestjs/mongoose": { + "optional": true + }, + "@nestjs/sequelize": { + "optional": true + }, + "@nestjs/typeorm": { + "optional": true + }, + "@prisma/client": { + "optional": true + }, + "mongoose": { + "optional": true + }, + "sequelize": { + "optional": true + }, + "typeorm": { + "optional": true + } + } + }, "node_modules/@nestjs/testing": { "version": "10.4.15", "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.4.15.tgz", @@ -2609,6 +2693,17 @@ } } }, + "node_modules/@nestjs/throttler": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@nestjs/throttler/-/throttler-6.4.0.tgz", + "integrity": "sha512-osL67i0PUuwU5nqSuJjtUJZMkxAnYB4VldgYUMGzvYRJDCqGRFMWbsbzm/CkUtPLRL30I8T74Xgt/OQxnYokiA==", + "license": "MIT", + "peerDependencies": { + "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", + "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", + "reflect-metadata": "^0.1.13 || ^0.2.0" + } + }, "node_modules/@nestjs/typeorm": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/@nestjs/typeorm/-/typeorm-10.0.2.tgz", @@ -3743,6 +3838,15 @@ "ajv": "^8.8.2" } }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "license": "ISC", + "dependencies": { + "string-width": "^4.1.0" + } + }, "node_modules/ansi-colors": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", @@ -4407,6 +4511,57 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/boxen": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-5.1.2.tgz", + "integrity": "sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ==", + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.0", + "camelcase": "^6.2.0", + "chalk": "^4.1.0", + "cli-boxes": "^2.2.1", + "string-width": "^4.2.2", + "type-fest": "^0.20.2", + "widest-line": "^3.1.0", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/boxen/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/boxen/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -4670,6 +4825,15 @@ "dev": true, "license": "MIT" }, + "node_modules/check-disk-space": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/check-disk-space/-/check-disk-space-3.4.0.tgz", + "integrity": "sha512-drVkSqfwA+TvuEhFipiR1OC9boEGZL5RrWvVsOthdcvQNXyCCuKkEiTOTXZ7qxSf/GLwq4GvzfrQD/Wz325hgw==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -4745,6 +4909,18 @@ "validator": "^13.9.0" } }, + "node_modules/cli-boxes": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", + "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -12903,7 +13079,6 @@ "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=10" @@ -13672,6 +13847,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "license": "MIT", + "dependencies": { + "string-width": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/winston": { "version": "3.17.0", "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", diff --git a/package.json b/package.json index 820dc5b..7e535ed 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@fast-csv/format": "^5.0.2", + "@nestjs/axios": "^4.0.0", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.2.0", "@nestjs/core": "^10.0.0", @@ -29,6 +30,8 @@ "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.0.0", "@nestjs/swagger": "^7.3.0", + "@nestjs/terminus": "^11.0.0", + "@nestjs/throttler": "^6.4.0", "@nestjs/typeorm": "^10.0.2", "@nestjs/websockets": "^10.3.8", "@tuya/tuya-connector-nodejs": "^2.1.2", diff --git a/src/app.module.ts b/src/app.module.ts index f07ff20..28e6156 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -12,7 +12,7 @@ import { UserNotificationModule } from './user-notification/user-notification.mo import { DeviceMessagesSubscriptionModule } from './device-messages/device-messages.module'; import { SceneModule } from './scene/scene.module'; import { DoorLockModule } from './door-lock/door.lock.module'; -import { APP_INTERCEPTOR } from '@nestjs/core'; +import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core'; import { LoggingInterceptor } from './interceptors/logging.interceptor'; import { AutomationModule } from './automation/automation.module'; import { RegionModule } from './region/region.module'; @@ -34,11 +34,17 @@ import { DeviceCommissionModule } from './commission-device/commission-device.mo import { PowerClampModule } from './power-clamp/power-clamp.module'; import { WinstonModule } from 'nest-winston'; import { winstonLoggerOptions } from './common/filters/http-exception/logger/winston.logger'; +import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler'; +import { HealthModule } from './health/health.module'; + @Module({ imports: [ ConfigModule.forRoot({ load: config, }), + ThrottlerModule.forRoot({ + throttlers: [{ ttl: 60000, limit: 10 }], + }), WinstonModule.forRoot(winstonLoggerOptions), ClientModule, AuthenticationModule, @@ -70,12 +76,17 @@ import { winstonLoggerOptions } from './common/filters/http-exception/logger/win TagModule, DeviceCommissionModule, PowerClampModule, + HealthModule, ], providers: [ { provide: APP_INTERCEPTOR, useClass: LoggingInterceptor, }, + { + provide: APP_GUARD, + useClass: ThrottlerGuard, + }, ], }) export class AppModule {} diff --git a/src/health/controllers/health.controller.ts b/src/health/controllers/health.controller.ts new file mode 100644 index 0000000..c7238ae --- /dev/null +++ b/src/health/controllers/health.controller.ts @@ -0,0 +1,35 @@ +import { Controller, Get } from '@nestjs/common'; +import { + HealthCheck, + HealthCheckService, + TypeOrmHealthIndicator, + DiskHealthIndicator, + MemoryHealthIndicator, + HttpHealthIndicator, +} from '@nestjs/terminus'; + +@Controller('health') +export class HealthController { + constructor( + private health: HealthCheckService, + private db: TypeOrmHealthIndicator, + private memory: MemoryHealthIndicator, + private disk: DiskHealthIndicator, + private http: HttpHealthIndicator, + ) {} + + @Get() + @HealthCheck() + check() { + return this.health.check([ + () => this.db.pingCheck('database'), + () => + this.disk.checkStorage('disk', { + thresholdPercent: 0.9, + path: '/', + }), + () => this.memory.checkHeap('memory_heap', 300 * 1024 * 1024), + () => this.http.pingCheck('tuya', 'https://openapi.tuya.com'), + ]); + } +} diff --git a/src/health/controllers/index.ts b/src/health/controllers/index.ts new file mode 100644 index 0000000..6b3bc30 --- /dev/null +++ b/src/health/controllers/index.ts @@ -0,0 +1 @@ +export * from './health.controller'; diff --git a/src/health/health.module.ts b/src/health/health.module.ts new file mode 100644 index 0000000..de6c36a --- /dev/null +++ b/src/health/health.module.ts @@ -0,0 +1,11 @@ +// src/health/health.module.ts +import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { HealthController } from './controllers'; + +@Module({ + imports: [TerminusModule, TypeOrmModule], + controllers: [HealthController], +}) +export class HealthModule {} From 0749fab63393a463727adbfe69e2787dee1f6119 Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Wed, 23 Apr 2025 17:31:16 +0400 Subject: [PATCH 4/9] health check --- libs/common/src/database/database.module.ts | 133 +++++++++++--------- src/health/controllers/health.controller.ts | 4 +- src/health/health.module.ts | 4 +- 3 files changed, 77 insertions(+), 64 deletions(-) diff --git a/libs/common/src/database/database.module.ts b/libs/common/src/database/database.module.ts index a25cf35..3b33ba4 100644 --- a/libs/common/src/database/database.module.ts +++ b/libs/common/src/database/database.module.ts @@ -43,73 +43,84 @@ import { SubspaceProductAllocationEntity } from '../modules/space/entities/subsp import { SubspaceEntity } from '../modules/space/entities/subspace/subspace.entity'; import { TagEntity } from '../modules/space/entities/tag.entity'; import { ClientEntity } from '../modules/client/entities'; +import { createLogger } from 'winston'; +import { winstonLoggerOptions } from 'src/common/filters/http-exception/logger/winston.logger'; + @Module({ imports: [ TypeOrmModule.forRootAsync({ imports: [ConfigModule], inject: [ConfigService], - useFactory: (configService: ConfigService) => ({ - name: 'default', - type: 'postgres', - host: configService.get('DB_HOST'), - port: configService.get('DB_PORT'), - username: configService.get('DB_USER'), - password: configService.get('DB_PASSWORD'), - database: configService.get('DB_NAME'), - entities: [ - NewTagEntity, - ProjectEntity, - UserEntity, - UserSessionEntity, - UserOtpEntity, - ProductEntity, - DeviceUserPermissionEntity, - DeviceEntity, - PermissionTypeEntity, - CommunityEntity, - SpaceEntity, - SpaceLinkEntity, - SubspaceEntity, - TagEntity, - UserSpaceEntity, - DeviceUserPermissionEntity, - RoleTypeEntity, - UserNotificationEntity, - DeviceNotificationEntity, - RegionEntity, - TimeZoneEntity, - VisitorPasswordEntity, - DeviceStatusLogEntity, - SceneEntity, - SceneIconEntity, - SceneDeviceEntity, - SpaceModelEntity, - SubspaceModelEntity, - TagModel, - InviteUserEntity, - InviteUserSpaceEntity, - InviteSpaceEntity, - AutomationEntity, - SpaceModelProductAllocationEntity, - SubspaceModelProductAllocationEntity, - SpaceProductAllocationEntity, - SubspaceProductAllocationEntity, - ClientEntity, - ], - namingStrategy: new SnakeNamingStrategy(), - synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))), - logging: false, - extra: { - charset: 'utf8mb4', - max: 20, // set pool max size - idleTimeoutMillis: 5000, // close idle clients after 5 second - connectionTimeoutMillis: 11_000, // return an error after 11 second if connection could not be established - maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion) - }, - continuationLocalStorage: true, - ssl: Boolean(JSON.parse(configService.get('DB_SSL'))), - }), + useFactory: (configService: ConfigService) => { + const sslEnabled = JSON.parse(configService.get('DB_SSL')); + const winstonLogger = createLogger(winstonLoggerOptions); + const typeOrmLogger = new TypeOrmWinstonLogger(winstonLogger); + + return { + name: 'default', + type: 'postgres', + host: configService.get('DB_HOST'), + port: configService.get('DB_PORT'), + username: configService.get('DB_USER'), + password: configService.get('DB_PASSWORD'), + database: configService.get('DB_NAME'), + synchronize: Boolean(JSON.parse(configService.get('DB_SYNC'))), + logging: ['query', 'error', 'warn', 'schema', 'migration'], + namingStrategy: new SnakeNamingStrategy(), + ssl: sslEnabled ? { rejectUnauthorized: false } : false, + extra: { + charset: 'utf8mb4', + max: 20, + idleTimeoutMillis: 5000, + connectionTimeoutMillis: 11000, + maxUses: 7500, + ...(sslEnabled && { + ssl: { rejectUnauthorized: false }, // Required for Azure PostgreSQL + }), + }, + entities: [ + NewTagEntity, + ProjectEntity, + UserEntity, + UserSessionEntity, + UserOtpEntity, + ProductEntity, + DeviceUserPermissionEntity, + DeviceEntity, + PermissionTypeEntity, + CommunityEntity, + SpaceEntity, + SpaceLinkEntity, + SubspaceEntity, + TagEntity, + UserSpaceEntity, + RoleTypeEntity, + UserNotificationEntity, + DeviceNotificationEntity, + RegionEntity, + TimeZoneEntity, + VisitorPasswordEntity, + DeviceStatusLogEntity, + SceneEntity, + SceneIconEntity, + SceneDeviceEntity, + SpaceModelEntity, + SubspaceModelEntity, + TagModel, + InviteUserEntity, + InviteUserSpaceEntity, + InviteSpaceEntity, + AutomationEntity, + SpaceModelProductAllocationEntity, + SubspaceModelProductAllocationEntity, + SpaceProductAllocationEntity, + SubspaceProductAllocationEntity, + ClientEntity, + ], + }; + }, }), ], + providers: [TypeOrmWinstonLogger], }) export class DatabaseModule {} diff --git a/src/health/controllers/health.controller.ts b/src/health/controllers/health.controller.ts index c7238ae..a16c527 100644 --- a/src/health/controllers/health.controller.ts +++ b/src/health/controllers/health.controller.ts @@ -1,4 +1,5 @@ import { Controller, Get } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; import { HealthCheck, HealthCheckService, @@ -8,6 +9,7 @@ import { HttpHealthIndicator, } from '@nestjs/terminus'; +@ApiTags('Health Module') @Controller('health') export class HealthController { constructor( @@ -29,7 +31,7 @@ export class HealthController { path: '/', }), () => this.memory.checkHeap('memory_heap', 300 * 1024 * 1024), - () => this.http.pingCheck('tuya', 'https://openapi.tuya.com'), + () => this.http.pingCheck('tuya', process.env.TUYA_EU_URL), ]); } } diff --git a/src/health/health.module.ts b/src/health/health.module.ts index de6c36a..2d5e6d1 100644 --- a/src/health/health.module.ts +++ b/src/health/health.module.ts @@ -3,9 +3,9 @@ import { Module } from '@nestjs/common'; import { TerminusModule } from '@nestjs/terminus'; import { TypeOrmModule } from '@nestjs/typeorm'; import { HealthController } from './controllers'; - +import { HttpModule } from '@nestjs/axios'; @Module({ - imports: [TerminusModule, TypeOrmModule], + imports: [TerminusModule, HttpModule, TypeOrmModule], controllers: [HealthController], }) export class HealthModule {} From fa3f49af18cafd825574b8f094c1873f29ae71db Mon Sep 17 00:00:00 2001 From: faris Aljohari <83524184+farisaljohari@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:06:41 +0300 Subject: [PATCH 5/9] fix: handle potential undefined deviceDetails in getDeviceByDeviceUuid --- src/device/services/device.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/device/services/device.service.ts b/src/device/services/device.service.ts index e00858c..9b78de5 100644 --- a/src/device/services/device.service.ts +++ b/src/device/services/device.service.ts @@ -685,7 +685,7 @@ export class DeviceService { return { ...rest, productUuid: product.uuid, - name: deviceDetails.name, + name: deviceDetails?.name, productName: product.name, } as GetDeviceDetailsInterface; } catch (error) { From 51bf4c01700814ec820e3a01da056e6912d9af4c Mon Sep 17 00:00:00 2001 From: faris Aljohari <83524184+farisaljohari@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:06:47 +0300 Subject: [PATCH 6/9] fix: update connection timeout value in database module --- libs/common/src/database/database.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/common/src/database/database.module.ts b/libs/common/src/database/database.module.ts index a25cf35..76a2d68 100644 --- a/libs/common/src/database/database.module.ts +++ b/libs/common/src/database/database.module.ts @@ -103,7 +103,7 @@ import { ClientEntity } from '../modules/client/entities'; charset: 'utf8mb4', max: 20, // set pool max size idleTimeoutMillis: 5000, // close idle clients after 5 second - connectionTimeoutMillis: 11_000, // return an error after 11 second if connection could not be established + connectionTimeoutMillis: 12_000, // return an error after 11 second if connection could not be established maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion) }, continuationLocalStorage: true, From 9844e13f4204a8196ed7958a5fc7016c78f68932 Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Thu, 24 Apr 2025 12:22:03 +0400 Subject: [PATCH 7/9] added health check --- src/health/health.module.ts | 2 ++ src/health/services/health.service.ts | 33 +++++++++++++++++++++++++++ src/health/services/index.ts | 1 + 3 files changed, 36 insertions(+) create mode 100644 src/health/services/health.service.ts create mode 100644 src/health/services/index.ts diff --git a/src/health/health.module.ts b/src/health/health.module.ts index 2d5e6d1..b1a2629 100644 --- a/src/health/health.module.ts +++ b/src/health/health.module.ts @@ -4,8 +4,10 @@ import { TerminusModule } from '@nestjs/terminus'; import { TypeOrmModule } from '@nestjs/typeorm'; import { HealthController } from './controllers'; import { HttpModule } from '@nestjs/axios'; +import { HealthService } from './services'; @Module({ imports: [TerminusModule, HttpModule, TypeOrmModule], controllers: [HealthController], + providers: [HealthService], }) export class HealthModule {} diff --git a/src/health/services/health.service.ts b/src/health/services/health.service.ts new file mode 100644 index 0000000..a330262 --- /dev/null +++ b/src/health/services/health.service.ts @@ -0,0 +1,33 @@ +import { Injectable } from '@nestjs/common'; +import { + HealthCheckService, + TypeOrmHealthIndicator, + DiskHealthIndicator, + MemoryHealthIndicator, + HttpHealthIndicator, + HealthCheckResult, +} from '@nestjs/terminus'; + +@Injectable() +export class HealthService { + constructor( + private health: HealthCheckService, + private db: TypeOrmHealthIndicator, + private memory: MemoryHealthIndicator, + private disk: DiskHealthIndicator, + private http: HttpHealthIndicator, + ) {} + + check(): Promise { + return this.health.check([ + () => this.db.pingCheck('database'), + () => + this.disk.checkStorage('disk', { + thresholdPercent: 0.9, + path: '/', + }), + () => this.memory.checkHeap('memory_heap', 300 * 1024 * 1024), + () => this.http.pingCheck('tuya', process.env.TUYA_EU_URL), + ]); + } +} diff --git a/src/health/services/index.ts b/src/health/services/index.ts new file mode 100644 index 0000000..1e2e6d7 --- /dev/null +++ b/src/health/services/index.ts @@ -0,0 +1 @@ +export * from './health.service'; From f9e1219f55c4ed80cd6386d2a94a62884822d4ff Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Thu, 24 Apr 2025 12:55:49 +0400 Subject: [PATCH 8/9] throw appropriate error --- src/device/services/device.service.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/device/services/device.service.ts b/src/device/services/device.service.ts index 9b78de5..59cee2f 100644 --- a/src/device/services/device.service.ts +++ b/src/device/services/device.service.ts @@ -582,6 +582,9 @@ export class DeviceService { return { successResults, failedResults }; } catch (error) { + if (error instanceof HttpException) { + throw error; + } throw new HttpException( error.message || 'Device Not Found', error.status || HttpStatus.NOT_FOUND, @@ -1116,6 +1119,9 @@ export class DeviceService { }); } } catch (error) { + if (error instanceof HttpException) { + throw error; + } throw new HttpException( error.message || 'Internal server error', HttpStatus.INTERNAL_SERVER_ERROR, From 96b911bfa12bdc7d6986948d0c04d751deff0215 Mon Sep 17 00:00:00 2001 From: hannathkadher Date: Thu, 24 Apr 2025 13:32:07 +0400 Subject: [PATCH 9/9] increase limit --- src/app.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.module.ts b/src/app.module.ts index ffb34f1..aa94a27 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -43,7 +43,7 @@ import { winstonLoggerOptions } from '../libs/common/src/logger/services/winston load: config, }), ThrottlerModule.forRoot({ - throttlers: [{ ttl: 60000, limit: 10 }], + throttlers: [{ ttl: 100000, limit: 30 }], }), WinstonModule.forRoot(winstonLoggerOptions), ClientModule,