Compare commits

..

31 Commits

Author SHA1 Message Date
4ff62611fb Merge branch 'dev' into test/prevent-server-block-on-rate-limit 2025-06-24 14:55:39 +03:00
7d9fe170b9 test: new format of IP 2025-06-24 14:53:04 +03:00
f337e6c681 Test/prevent server block on rate limit (#421) 2025-06-24 10:55:38 +03:00
5ad5e7e934 test: use forwarded for heder to catch real IP 2025-06-24 10:46:14 +03:00
f5bf857071 Merge pull request #429 from SyncrowIOT/add-queue-event-handler
Add queue event handler
2025-06-23 08:13:36 -06:00
d1d4d529a8 Add methods to handle SOS events and device status updates in Firebase and our DB 2025-06-23 08:10:33 -06:00
37b582f521 Merge pull request #428 from SyncrowIOT/add-queue-event-handler
Implement message queue for TuyaWebSocketService and batch processing
2025-06-23 07:35:22 -06:00
cf19f08dca turn on all the updates data points 2025-06-23 07:33:01 -06:00
ff370b2baa Implement message queue for TuyaWebSocketService and batch processing 2025-06-23 07:31:58 -06:00
04f64407e1 turn off some update data points 2025-06-23 07:10:47 -06:00
d7eef5d03e Merge pull request #427 from SyncrowIOT/revert-426-SP-1778-be-fix-time-out-connections-in-the-db
Revert "SP-1778-be-fix-time-out-connections-in-the-db"
2025-06-23 07:09:20 -06:00
c8d691b380 tern off data procedure 2025-06-23 07:02:23 -06:00
75d03366c2 Revert "SP-1778-be-fix-time-out-connections-in-the-db" 2025-06-23 06:58:57 -06:00
52cb69cc84 Merge pull request #426 from SyncrowIOT/SP-1778-be-fix-time-out-connections-in-the-db
SP-1778-be-fix-time-out-connections-in-the-db
2025-06-23 06:38:58 -06:00
a6053b3971 refactor: implement query runners for database operations in multiple services 2025-06-23 06:34:53 -06:00
60d2c8330b fix: increase DB max pool size (#425) 2025-06-23 15:23:53 +03:00
fddd06e06d fix: add space condition to the join operator instead of general query (#423) 2025-06-23 12:44:19 +03:00
3160773c2a fix: spaces structure in communities (#420) 2025-06-23 10:21:55 +03:00
110ed4157a task: add spaces filter to get devices by project (#422) 2025-06-23 09:34:59 +03:00
238a52bfa9 Merge branch 'dev' into test/prevent-server-block-on-rate-limit 2025-06-22 14:07:09 +03:00
f2a8ed141c test throttle module 2025-06-22 10:23:37 +03:00
bf64470288 increase DB max connection to 50 2025-06-19 14:31:16 +03:00
c1e9c6cbb7 Merge branch 'dev' into test/prevent-server-block-on-rate-limit 2025-06-19 13:42:55 +03:00
f28184975f revert all changes 2025-06-19 13:37:26 +03:00
130a1ed06e fix: merge conflicts 2025-06-19 12:53:34 +03:00
01d66a67d9 Merge branch 'dev' into test/prevent-server-block-on-rate-limit 2025-06-19 10:13:38 +03:00
069db9a3ea task: increase rate limit timeout 2025-06-19 10:11:51 +03:00
ce1986a27f Merge branch 'dev' into test/prevent-server-block-on-rate-limit 2025-06-19 09:44:01 +03:00
6857b4ea03 task: test rate limits on sever 2025-06-19 09:42:11 +03:00
e6c3fc7044 add logging 2025-06-18 12:05:09 +03:00
588eacdfef task: add trust proxy header 2025-06-18 12:00:27 +03:00
10 changed files with 170 additions and 65 deletions

View File

@ -125,7 +125,7 @@ import { VisitorPasswordEntity } from '../modules/visitor-password/entities';
logger: typeOrmLogger, logger: typeOrmLogger,
extra: { extra: {
charset: 'utf8mb4', charset: 'utf8mb4',
max: 50, // set pool max size max: 100, // set pool max size
idleTimeoutMillis: 5000, // close idle clients after 5 second idleTimeoutMillis: 5000, // close idle clients after 5 second
connectionTimeoutMillis: 12_000, // return an error after 11 second if connection could not be established connectionTimeoutMillis: 12_000, // return an error after 11 second if connection could not be established
maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion) maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion)

View File

@ -76,6 +76,28 @@ export class DeviceStatusFirebaseService {
); );
} }
} }
async addDeviceStatusToOurDb(
addDeviceStatusDto: AddDeviceStatusDto,
): Promise<AddDeviceStatusDto | null> {
try {
const device = await this.getDeviceByDeviceTuyaUuid(
addDeviceStatusDto.deviceTuyaUuid,
);
if (device?.uuid) {
return await this.createDeviceStatusInOurDb({
deviceUuid: device.uuid,
...addDeviceStatusDto,
productType: device.productDevice.prodType,
});
}
// Return null if device not found or no UUID
return null;
} catch (error) {
// Handle the error silently, perhaps log it internally or ignore it
return null;
}
}
async addDeviceStatusToFirebase( async addDeviceStatusToFirebase(
addDeviceStatusDto: AddDeviceStatusDto, addDeviceStatusDto: AddDeviceStatusDto,
): Promise<AddDeviceStatusDto | null> { ): Promise<AddDeviceStatusDto | null> {
@ -211,6 +233,13 @@ export class DeviceStatusFirebaseService {
return existingData; return existingData;
}); });
// Return the updated data
const snapshot: DataSnapshot = await get(dataRef);
return snapshot.val();
}
async createDeviceStatusInOurDb(
addDeviceStatusDto: AddDeviceStatusDto,
): Promise<any> {
// Save logs to your repository // Save logs to your repository
const newLogs = addDeviceStatusDto.log.properties.map((property) => { const newLogs = addDeviceStatusDto.log.properties.map((property) => {
return this.deviceStatusLogRepository.create({ return this.deviceStatusLogRepository.create({
@ -269,8 +298,5 @@ export class DeviceStatusFirebaseService {
addDeviceStatusDto.deviceUuid, addDeviceStatusDto.deviceUuid,
); );
} }
// Return the updated data
const snapshot: DataSnapshot = await get(dataRef);
return snapshot.val();
} }
} }

View File

@ -16,7 +16,7 @@ export class SosHandlerService {
); );
} }
async handleSosEvent(devId: string, logData: any): Promise<void> { async handleSosEventFirebase(devId: string, logData: any): Promise<void> {
try { try {
await this.deviceStatusFirebaseService.addDeviceStatusToFirebase({ await this.deviceStatusFirebaseService.addDeviceStatusToFirebase({
deviceTuyaUuid: devId, deviceTuyaUuid: devId,
@ -39,4 +39,28 @@ export class SosHandlerService {
this.logger.error('Failed to send SOS true value', err); this.logger.error('Failed to send SOS true value', err);
} }
} }
async handleSosEventOurDb(devId: string, logData: any): Promise<void> {
try {
await this.deviceStatusFirebaseService.addDeviceStatusToOurDb({
deviceTuyaUuid: devId,
status: [{ code: 'sos', value: true }],
log: logData,
});
setTimeout(async () => {
try {
await this.deviceStatusFirebaseService.addDeviceStatusToOurDb({
deviceTuyaUuid: devId,
status: [{ code: 'sos', value: false }],
log: logData,
});
} catch (err) {
this.logger.error('Failed to send SOS false value', err);
}
}, 2000);
} catch (err) {
this.logger.error('Failed to send SOS true value', err);
}
}
} }

View File

@ -9,6 +9,14 @@ export class TuyaWebSocketService {
private client: any; private client: any;
private readonly isDevEnv: boolean; private readonly isDevEnv: boolean;
private messageQueue: {
devId: string;
status: any;
logData: any;
}[] = [];
private isProcessing = false;
constructor( constructor(
private readonly configService: ConfigService, private readonly configService: ConfigService,
private readonly deviceStatusFirebaseService: DeviceStatusFirebaseService, private readonly deviceStatusFirebaseService: DeviceStatusFirebaseService,
@ -26,12 +34,12 @@ export class TuyaWebSocketService {
}); });
if (this.configService.get<string>('tuya-config.TRUN_ON_TUYA_SOCKET')) { if (this.configService.get<string>('tuya-config.TRUN_ON_TUYA_SOCKET')) {
// Set up event handlers
this.setupEventHandlers(); this.setupEventHandlers();
// Start receiving messages
this.client.start(); this.client.start();
} }
// Trigger the queue processor every 2 seconds
setInterval(() => this.processQueue(), 10000);
} }
private setupEventHandlers() { private setupEventHandlers() {
@ -43,10 +51,10 @@ export class TuyaWebSocketService {
this.client.message(async (ws: WebSocket, message: any) => { this.client.message(async (ws: WebSocket, message: any) => {
try { try {
const { devId, status, logData } = this.extractMessageData(message); const { devId, status, logData } = this.extractMessageData(message);
if (this.sosHandlerService.isSosTriggered(status)) { if (this.sosHandlerService.isSosTriggered(status)) {
await this.sosHandlerService.handleSosEvent(devId, logData); await this.sosHandlerService.handleSosEventFirebase(devId, logData);
} else { } else {
// Firebase real-time update
await this.deviceStatusFirebaseService.addDeviceStatusToFirebase({ await this.deviceStatusFirebaseService.addDeviceStatusToFirebase({
deviceTuyaUuid: devId, deviceTuyaUuid: devId,
status: status, status: status,
@ -54,9 +62,13 @@ export class TuyaWebSocketService {
}); });
} }
// Push to internal queue
this.messageQueue.push({ devId, status, logData });
// Acknowledge the message
this.client.ackMessage(message.messageId); this.client.ackMessage(message.messageId);
} catch (error) { } catch (error) {
console.error('Error processing message:', error); console.error('Error receiving message:', error);
} }
}); });
@ -80,6 +92,38 @@ export class TuyaWebSocketService {
console.error('WebSocket error:', error); console.error('WebSocket error:', error);
}); });
} }
private async processQueue() {
if (this.isProcessing || this.messageQueue.length === 0) return;
this.isProcessing = true;
const batch = [...this.messageQueue];
this.messageQueue = [];
try {
for (const item of batch) {
if (this.sosHandlerService.isSosTriggered(item.status)) {
await this.sosHandlerService.handleSosEventOurDb(
item.devId,
item.logData,
);
} else {
await this.deviceStatusFirebaseService.addDeviceStatusToOurDb({
deviceTuyaUuid: item.devId,
status: item.status,
log: item.logData,
});
}
}
} catch (error) {
console.error('Error processing batch:', error);
// Re-add the batch to the queue for retry
this.messageQueue.unshift(...batch);
} finally {
this.isProcessing = false;
}
}
private extractMessageData(message: any): { private extractMessageData(message: any): {
devId: string; devId: string;
status: any; status: any;

View File

@ -1,7 +1,7 @@
import { SeederModule } from '@app/common/seed/seeder.module'; import { SeederModule } from '@app/common/seed/seeder.module';
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config'; import { ConfigModule } from '@nestjs/config';
import { APP_INTERCEPTOR } from '@nestjs/core'; import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core';
import { WinstonModule } from 'nest-winston'; import { WinstonModule } from 'nest-winston';
import { AuthenticationModule } from './auth/auth.module'; import { AuthenticationModule } from './auth/auth.module';
import { AutomationModule } from './automation/automation.module'; import { AutomationModule } from './automation/automation.module';
@ -35,6 +35,9 @@ import { UserNotificationModule } from './user-notification/user-notification.mo
import { UserModule } from './users/user.module'; import { UserModule } from './users/user.module';
import { VisitorPasswordModule } from './vistor-password/visitor-password.module'; import { VisitorPasswordModule } from './vistor-password/visitor-password.module';
import { ThrottlerGuard } from '@nestjs/throttler';
import { ThrottlerModule } from '@nestjs/throttler/dist/throttler.module';
import { isArray } from 'class-validator';
import { winstonLoggerOptions } from '../libs/common/src/logger/services/winston.logger'; import { winstonLoggerOptions } from '../libs/common/src/logger/services/winston.logger';
import { AqiModule } from './aqi/aqi.module'; import { AqiModule } from './aqi/aqi.module';
import { OccupancyModule } from './occupancy/occupancy.module'; import { OccupancyModule } from './occupancy/occupancy.module';
@ -44,9 +47,18 @@ import { WeatherModule } from './weather/weather.module';
ConfigModule.forRoot({ ConfigModule.forRoot({
load: config, load: config,
}), }),
/* ThrottlerModule.forRoot({ ThrottlerModule.forRoot({
throttlers: [{ ttl: 100000, limit: 30 }], throttlers: [{ ttl: 60000, limit: 30 }],
}), */ generateKey: (context) => {
const req = context.switchToHttp().getRequest();
console.log('Real IP:', req.headers['x-forwarded-for']);
return req.headers['x-forwarded-for']
? isArray(req.headers['x-forwarded-for'])
? req.headers['x-forwarded-for'][0].split(':')[0]
: req.headers['x-forwarded-for'].split(':')[0]
: req.ip;
},
}),
WinstonModule.forRoot(winstonLoggerOptions), WinstonModule.forRoot(winstonLoggerOptions),
ClientModule, ClientModule,
AuthenticationModule, AuthenticationModule,
@ -88,10 +100,10 @@ import { WeatherModule } from './weather/weather.module';
provide: APP_INTERCEPTOR, provide: APP_INTERCEPTOR,
useClass: LoggingInterceptor, useClass: LoggingInterceptor,
}, },
/* { {
provide: APP_GUARD, provide: APP_GUARD,
useClass: ThrottlerGuard, useClass: ThrottlerGuard,
}, */ },
], ],
}) })
export class AppModule {} export class AppModule {}

View File

@ -190,25 +190,26 @@ export class CommunityService {
.distinct(true); .distinct(true);
if (includeSpaces) { if (includeSpaces) {
qb.leftJoinAndSelect('c.spaces', 'space', 'space.disabled = false') qb.leftJoinAndSelect(
'c.spaces',
'space',
'space.disabled = :disabled AND space.spaceName != :orphanSpaceName',
{ disabled: false, orphanSpaceName: ORPHAN_SPACE_NAME },
)
.leftJoinAndSelect('space.parent', 'parent') .leftJoinAndSelect('space.parent', 'parent')
.leftJoinAndSelect( .leftJoinAndSelect(
'space.children', 'space.children',
'children', 'children',
'children.disabled = :disabled', 'children.disabled = :disabled',
{ disabled: false }, { disabled: false },
) );
// .leftJoinAndSelect('space.spaceModel', 'spaceModel') // .leftJoinAndSelect('space.spaceModel', 'spaceModel')
.andWhere('space.spaceName != :orphanSpaceName', {
orphanSpaceName: ORPHAN_SPACE_NAME,
})
.andWhere('space.disabled = :disabled', { disabled: false });
} }
if (search) { if (search) {
qb.andWhere( qb.andWhere(
`c.name ILIKE :search ${includeSpaces ? 'OR space.space_name ILIKE :search' : ''}`, `c.name ILIKE :search ${includeSpaces ? 'OR space.space_name ILIKE :search' : ''}`,
{ search: `%${search}%` }, { search },
); );
} }
@ -216,7 +217,6 @@ export class CommunityService {
const { baseResponseDto, paginationResponseDto } = const { baseResponseDto, paginationResponseDto } =
await customModel.findAll({ ...pageable, modelName: 'community' }, qb); await customModel.findAll({ ...pageable, modelName: 'community' }, qb);
if (includeSpaces) { if (includeSpaces) {
baseResponseDto.data = baseResponseDto.data.map((community) => ({ baseResponseDto.data = baseResponseDto.data.map((community) => ({
...community, ...community,

View File

@ -1,11 +1,11 @@
import { DeviceService } from '../services/device.service';
import { Controller, Get, Param, Query, UseGuards } from '@nestjs/common';
import { ApiTags, ApiBearerAuth, ApiOperation } from '@nestjs/swagger';
import { EnableDisableStatusEnum } from '@app/common/constants/days.enum';
import { ControllerRoute } from '@app/common/constants/controller-route'; import { ControllerRoute } from '@app/common/constants/controller-route';
import { PermissionsGuard } from 'src/guards/permissions.guard'; import { EnableDisableStatusEnum } from '@app/common/constants/days.enum';
import { Controller, Get, Param, Query, UseGuards } from '@nestjs/common';
import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
import { Permissions } from 'src/decorators/permissions.decorator'; import { Permissions } from 'src/decorators/permissions.decorator';
import { GetDoorLockDevices, ProjectParam } from '../dtos'; import { PermissionsGuard } from 'src/guards/permissions.guard';
import { GetDevicesFilterDto, ProjectParam } from '../dtos';
import { DeviceService } from '../services/device.service';
@ApiTags('Device Module') @ApiTags('Device Module')
@Controller({ @Controller({
@ -25,7 +25,7 @@ export class DeviceProjectController {
}) })
async getAllDevices( async getAllDevices(
@Param() param: ProjectParam, @Param() param: ProjectParam,
@Query() query: GetDoorLockDevices, @Query() query: GetDevicesFilterDto,
) { ) {
return await this.deviceService.getAllDevices(param, query); return await this.deviceService.getAllDevices(param, query);
} }

View File

@ -1,6 +1,7 @@
import { DeviceTypeEnum } from '@app/common/constants/device-type.enum'; import { DeviceTypeEnum } from '@app/common/constants/device-type.enum';
import { ApiProperty } from '@nestjs/swagger'; import { ApiProperty } from '@nestjs/swagger';
import { import {
IsArray,
IsEnum, IsEnum,
IsNotEmpty, IsNotEmpty,
IsOptional, IsOptional,
@ -41,16 +42,7 @@ export class GetDeviceLogsDto {
@IsOptional() @IsOptional()
public endTime: string; public endTime: string;
} }
export class GetDoorLockDevices {
@ApiProperty({
description: 'Device Type',
enum: DeviceTypeEnum,
required: false,
})
@IsEnum(DeviceTypeEnum)
@IsOptional()
public deviceType: DeviceTypeEnum;
}
export class GetDevicesBySpaceOrCommunityDto { export class GetDevicesBySpaceOrCommunityDto {
@ApiProperty({ @ApiProperty({
description: 'Device Product Type', description: 'Device Product Type',
@ -72,3 +64,23 @@ export class GetDevicesBySpaceOrCommunityDto {
@IsNotEmpty({ message: 'Either spaceUuid or communityUuid must be provided' }) @IsNotEmpty({ message: 'Either spaceUuid or communityUuid must be provided' })
requireEither?: never; // This ensures at least one of them is provided requireEither?: never; // This ensures at least one of them is provided
} }
export class GetDevicesFilterDto {
@ApiProperty({
description: 'Device Type',
enum: DeviceTypeEnum,
required: false,
})
@IsEnum(DeviceTypeEnum)
@IsOptional()
public deviceType: DeviceTypeEnum;
@ApiProperty({
description: 'List of Space IDs to filter devices',
required: false,
example: ['60d21b4667d0d8992e610c85', '60d21b4967d0d8992e610c86'],
})
@IsOptional()
@IsArray()
@IsUUID('4', { each: true })
public spaces?: string[];
}

View File

@ -53,7 +53,7 @@ import { DeviceSceneParamDto } from '../dtos/device.param.dto';
import { import {
GetDeviceLogsDto, GetDeviceLogsDto,
GetDevicesBySpaceOrCommunityDto, GetDevicesBySpaceOrCommunityDto,
GetDoorLockDevices, GetDevicesFilterDto,
} from '../dtos/get.device.dto'; } from '../dtos/get.device.dto';
import { import {
controlDeviceInterface, controlDeviceInterface,
@ -955,19 +955,20 @@ export class DeviceService {
async getAllDevices( async getAllDevices(
param: ProjectParam, param: ProjectParam,
query: GetDoorLockDevices, { deviceType, spaces }: GetDevicesFilterDto,
): Promise<BaseResponseDto> { ): Promise<BaseResponseDto> {
try { try {
await this.validateProject(param.projectUuid); await this.validateProject(param.projectUuid);
if (query.deviceType === DeviceTypeEnum.DOOR_LOCK) { if (deviceType === DeviceTypeEnum.DOOR_LOCK) {
return await this.getDoorLockDevices(param.projectUuid); return await this.getDoorLockDevices(param.projectUuid, spaces);
} else if (!query.deviceType) { } else if (!deviceType) {
const devices = await this.deviceRepository.find({ const devices = await this.deviceRepository.find({
where: { where: {
isActive: true, isActive: true,
spaceDevice: { spaceDevice: {
community: { project: { uuid: param.projectUuid } }, uuid: spaces && spaces.length ? In(spaces) : undefined,
spaceName: Not(ORPHAN_SPACE_NAME), spaceName: Not(ORPHAN_SPACE_NAME),
community: { project: { uuid: param.projectUuid } },
}, },
}, },
relations: [ relations: [
@ -1563,7 +1564,7 @@ export class DeviceService {
} }
} }
async getDoorLockDevices(projectUuid: string) { async getDoorLockDevices(projectUuid: string, spaces?: string[]) {
await this.validateProject(projectUuid); await this.validateProject(projectUuid);
const devices = await this.deviceRepository.find({ const devices = await this.deviceRepository.find({
@ -1573,6 +1574,7 @@ export class DeviceService {
}, },
spaceDevice: { spaceDevice: {
spaceName: Not(ORPHAN_SPACE_NAME), spaceName: Not(ORPHAN_SPACE_NAME),
uuid: spaces && spaces.length ? In(spaces) : undefined,
community: { community: {
project: { project: {
uuid: projectUuid, uuid: projectUuid,

View File

@ -3,7 +3,6 @@ import { SeederService } from '@app/common/seed/services/seeder.service';
import { Logger, ValidationPipe } from '@nestjs/common'; import { Logger, ValidationPipe } from '@nestjs/common';
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { json, urlencoded } from 'body-parser'; import { json, urlencoded } from 'body-parser';
import rateLimit from 'express-rate-limit';
import helmet from 'helmet'; import helmet from 'helmet';
import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston';
import { setupSwaggerAuthentication } from '../libs/common/src/util/user-auth.swagger.utils'; import { setupSwaggerAuthentication } from '../libs/common/src/util/user-auth.swagger.utils';
@ -22,20 +21,6 @@ async function bootstrap() {
app.use(new RequestContextMiddleware().use); app.use(new RequestContextMiddleware().use);
app.use(
rateLimit({
windowMs: 5 * 60 * 1000,
max: 500,
}),
);
app.use((req, res, next) => {
console.log('Real IP:', req.ip);
next();
});
// app.getHttpAdapter().getInstance().set('trust proxy', 1);
app.use( app.use(
helmet({ helmet({
contentSecurityPolicy: false, contentSecurityPolicy: false,