Compare commits

..

9 Commits

Author SHA1 Message Date
bf9294a4ef fix: increase DB max pool size 2025-06-23 15:19:11 +03:00
fddd06e06d fix: add space condition to the join operator instead of general query (#423) 2025-06-23 12:44:19 +03:00
3160773c2a fix: spaces structure in communities (#420) 2025-06-23 10:21:55 +03:00
110ed4157a task: add spaces filter to get devices by project (#422) 2025-06-23 09:34:59 +03:00
aa9e90bf08 Test/prevent server block on rate limit (#419)
* increase DB max connection to 50
2025-06-19 14:34:23 +03:00
c5dd5e28fd Test/prevent server block on rate limit (#418) 2025-06-19 13:54:22 +03:00
603e74af09 Test/prevent server block on rate limit (#417)
* task: add trust proxy header

* add logging

* task: test rate limits on sever

* task: increase rate limit timeout

* fix: merge conflicts
2025-06-19 12:54:59 +03:00
0e36f32ed6 Test/prevent server block on rate limit (#415)
* task: increase rate limit timeout
2025-06-19 10:15:29 +03:00
705ceeba29 Test/prevent server block on rate limit (#414)
* task: test rate limits on sever
2025-06-19 09:45:09 +03:00
10 changed files with 94 additions and 138 deletions

View File

@ -125,7 +125,7 @@ import { VisitorPasswordEntity } from '../modules/visitor-password/entities';
logger: typeOrmLogger, logger: typeOrmLogger,
extra: { extra: {
charset: 'utf8mb4', charset: 'utf8mb4',
max: 20, // set pool max size max: 100, // set pool max size
idleTimeoutMillis: 5000, // close idle clients after 5 second idleTimeoutMillis: 5000, // close idle clients after 5 second
connectionTimeoutMillis: 12_000, // return an error after 11 second if connection could not be established connectionTimeoutMillis: 12_000, // return an error after 11 second if connection could not be established
maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion) maxUses: 7500, // close (and replace) a connection after it has been used 7500 times (see below for discussion)

View File

@ -1,4 +1,3 @@
-- 1. Load all presence-related logs
WITH device_logs AS ( WITH device_logs AS (
SELECT SELECT
device.uuid AS device_id, device.uuid AS device_id,
@ -16,7 +15,7 @@ WITH device_logs AS (
AND "device-status-log".code = 'presence_state' AND "device-status-log".code = 'presence_state'
), ),
-- 2. Standard transitions: 'none' → 'motion' or 'presence' -- 1. All 'none' → presence or motion
presence_transitions AS ( presence_transitions AS (
SELECT SELECT
space_id, space_id,
@ -24,31 +23,10 @@ presence_transitions AS (
event_time::date AS event_date, event_time::date AS event_date,
value value
FROM device_logs FROM device_logs
WHERE value IN ('motion', 'presence') AND prev_value = 'none' WHERE (value = 'motion' OR value = 'presence') AND prev_value = 'none'
), ),
-- 3. Fallback: days with 'motion' or 'presence' but no 'none' -- 2. Cluster events per space_id within 30s
fallback_daily_presence AS (
SELECT
space_id,
event_time::date AS event_date,
MIN(event_time) AS event_time,
'presence'::text AS value
FROM device_logs
WHERE value IN ('motion', 'presence', 'none')
GROUP BY space_id, event_time::date
HAVING BOOL_OR(value = 'motion') OR BOOL_OR(value = 'presence')
AND NOT BOOL_OR(value = 'none')
),
-- 4. Merge both detection sources
all_presence_events AS (
SELECT * FROM presence_transitions
UNION ALL
SELECT space_id, event_time, event_date, value FROM fallback_daily_presence
),
-- 5. Cluster events per space_id within 30 seconds
clustered_events AS ( clustered_events AS (
SELECT SELECT
space_id, space_id,
@ -62,11 +40,11 @@ clustered_events AS (
WHEN event_time - LAG(event_time) OVER (PARTITION BY space_id ORDER BY event_time) > INTERVAL '30 seconds' WHEN event_time - LAG(event_time) OVER (PARTITION BY space_id ORDER BY event_time) > INTERVAL '30 seconds'
THEN 1 ELSE 0 THEN 1 ELSE 0
END AS new_cluster_flag END AS new_cluster_flag
FROM all_presence_events FROM presence_transitions
) marked ) marked
), ),
-- 6. Determine dominant type (motion vs presence) per cluster -- 3. Determine dominant type (motion vs presence) per cluster
cluster_type AS ( cluster_type AS (
SELECT SELECT
space_id, space_id,
@ -82,7 +60,7 @@ cluster_type AS (
GROUP BY space_id, event_date, cluster_id GROUP BY space_id, event_date, cluster_id
), ),
-- 7. Count clusters by dominant type -- 4. Count clusters by dominant type
summary AS ( summary AS (
SELECT SELECT
space_id, space_id,
@ -92,16 +70,15 @@ summary AS (
COUNT(*) AS count_total_presence_detected COUNT(*) AS count_total_presence_detected
FROM cluster_type FROM cluster_type
GROUP BY space_id, event_date GROUP BY space_id, event_date
),
-- 8. Prepare final result
final_table AS (
SELECT *
FROM summary
ORDER BY space_id, event_date
) )
-- 9. Insert or upsert into the destination table -- 5. Output
, final_table as (
SELECT *
FROM summary
ORDER BY space_id, event_date)
INSERT INTO public."presence-sensor-daily-space-detection" ( INSERT INTO public."presence-sensor-daily-space-detection" (
space_uuid, space_uuid,
event_date, event_date,
@ -120,4 +97,4 @@ ON CONFLICT (space_uuid, event_date) DO UPDATE
SET SET
count_motion_detected = EXCLUDED.count_motion_detected, count_motion_detected = EXCLUDED.count_motion_detected,
count_presence_detected = EXCLUDED.count_presence_detected, count_presence_detected = EXCLUDED.count_presence_detected,
count_total_presence_detected = EXCLUDED.count_total_presence_detected; count_total_presence_detected = EXCLUDED.count_total_presence_detected;

View File

@ -4,7 +4,6 @@ WITH params AS (
$2::uuid AS space_id $2::uuid AS space_id
), ),
-- 1. Load logs
device_logs AS ( device_logs AS (
SELECT SELECT
device.uuid AS device_id, device.uuid AS device_id,
@ -22,7 +21,7 @@ device_logs AS (
AND "device-status-log".code = 'presence_state' AND "device-status-log".code = 'presence_state'
), ),
-- 2. Transitions from 'none' → motion/presence -- 1. All 'none' → presence or motion
presence_transitions AS ( presence_transitions AS (
SELECT SELECT
space_id, space_id,
@ -30,30 +29,10 @@ presence_transitions AS (
event_time::date AS event_date, event_time::date AS event_date,
value value
FROM device_logs FROM device_logs
WHERE value IN ('motion', 'presence') AND prev_value = 'none' WHERE (value = 'motion' OR value = 'presence') AND prev_value = 'none'
), ),
-- 3. Fallback: days with motion/presence but no 'none' -- 2. Cluster events per space_id within 30s
fallback_daily_presence AS (
SELECT
space_id,
event_time::date AS event_date,
MIN(event_time) AS event_time,
'presence'::text AS value
FROM device_logs
GROUP BY space_id, event_time::date
HAVING BOOL_OR(value = 'motion') OR BOOL_OR(value = 'presence')
AND NOT BOOL_OR(value = 'none')
),
-- 4. Combine standard and fallback detections
all_presence_events AS (
SELECT * FROM presence_transitions
UNION ALL
SELECT * FROM fallback_daily_presence
),
-- 5. Cluster detections (within 30s)
clustered_events AS ( clustered_events AS (
SELECT SELECT
space_id, space_id,
@ -67,11 +46,11 @@ clustered_events AS (
WHEN event_time - LAG(event_time) OVER (PARTITION BY space_id ORDER BY event_time) > INTERVAL '30 seconds' WHEN event_time - LAG(event_time) OVER (PARTITION BY space_id ORDER BY event_time) > INTERVAL '30 seconds'
THEN 1 ELSE 0 THEN 1 ELSE 0
END AS new_cluster_flag END AS new_cluster_flag
FROM all_presence_events FROM presence_transitions
) marked ) marked
), ),
-- 6. Dominant type per cluster -- 3. Determine dominant type (motion vs presence) per cluster
cluster_type AS ( cluster_type AS (
SELECT SELECT
space_id, space_id,
@ -87,7 +66,7 @@ cluster_type AS (
GROUP BY space_id, event_date, cluster_id GROUP BY space_id, event_date, cluster_id
), ),
-- 7. Count presence by type -- 4. Count clusters by dominant type
summary AS ( summary AS (
SELECT SELECT
space_id, space_id,
@ -97,22 +76,22 @@ summary AS (
COUNT(*) AS count_total_presence_detected COUNT(*) AS count_total_presence_detected
FROM cluster_type FROM cluster_type
GROUP BY space_id, event_date GROUP BY space_id, event_date
),
-- 8. Filter by params and return final table
final_table AS (
SELECT
summary.space_id,
summary.event_date,
count_motion_detected,
count_presence_detected,
count_total_presence_detected
FROM summary
JOIN params p ON summary.space_id = p.space_id
WHERE p.event_date IS NULL OR summary.event_date = p.event_date
) )
-- 9. Insert or upsert into the table -- 5. Output
, final_table as (
SELECT summary.space_id,
summary.event_date,
count_motion_detected,
count_presence_detected,
count_total_presence_detected
FROM summary
JOIN params P ON true
where summary.space_id = P.space_id
and (P.event_date IS NULL or summary.event_date::date = P.event_date)
ORDER BY space_id, event_date)
INSERT INTO public."presence-sensor-daily-space-detection" ( INSERT INTO public."presence-sensor-daily-space-detection" (
space_uuid, space_uuid,
event_date, event_date,

View File

@ -30,19 +30,6 @@ presence_detection AS (
FROM device_logs FROM device_logs
), ),
fallback_daily_presence AS (
SELECT
space_id,
event_time::date AS event_date,
0 AS event_hour,
COUNT(*) > 0 AS has_presence,
BOOL_OR(value = 'none') AS has_none
FROM device_logs
WHERE value IN ('motion', 'presence', 'none')
GROUP BY space_id, event_time::date
HAVING COUNT(*) > 0 AND NOT BOOL_OR(value = 'none')
),
space_level_presence_events AS ( space_level_presence_events AS (
SELECT DISTINCT SELECT DISTINCT
pd.space_id, pd.space_id,
@ -51,15 +38,6 @@ space_level_presence_events AS (
pd.event_time pd.event_time
FROM presence_detection pd FROM presence_detection pd
WHERE presence_started = 1 WHERE presence_started = 1
UNION
SELECT
fdp.space_id,
fdp.event_date,
fdp.event_hour,
NULL::timestamp AS event_time
FROM fallback_daily_presence fdp
), ),
space_level_presence_summary AS ( space_level_presence_summary AS (
@ -99,4 +77,3 @@ LEFT JOIN space_level_presence_summary pds
ORDER BY space_id, event_date, event_hour; ORDER BY space_id, event_date, event_hour;

View File

@ -190,24 +190,26 @@ export class CommunityService {
.distinct(true); .distinct(true);
if (includeSpaces) { if (includeSpaces) {
qb.leftJoinAndSelect('c.spaces', 'space', 'space.disabled = false') qb.leftJoinAndSelect(
'c.spaces',
'space',
'space.disabled = :disabled AND space.spaceName != :orphanSpaceName',
{ disabled: false, orphanSpaceName: ORPHAN_SPACE_NAME },
)
.leftJoinAndSelect('space.parent', 'parent') .leftJoinAndSelect('space.parent', 'parent')
.leftJoinAndSelect( .leftJoinAndSelect(
'space.children', 'space.children',
'children', 'children',
'children.disabled = :disabled', 'children.disabled = :disabled',
{ disabled: false }, { disabled: false },
) );
// .leftJoinAndSelect('space.spaceModel', 'spaceModel') // .leftJoinAndSelect('space.spaceModel', 'spaceModel')
.andWhere('space.spaceName != :orphanSpaceName', {
orphanSpaceName: ORPHAN_SPACE_NAME,
})
.andWhere('space.disabled = :disabled', { disabled: false });
} }
if (search) { if (search) {
qb.andWhere( qb.andWhere(
`c.name ILIKE '%${search}%' ${includeSpaces ? "OR space.space_name ILIKE '%" + search + "%'" : ''}`, `c.name ILIKE :search ${includeSpaces ? 'OR space.space_name ILIKE :search' : ''}`,
{ search },
); );
} }
@ -215,12 +217,21 @@ export class CommunityService {
const { baseResponseDto, paginationResponseDto } = const { baseResponseDto, paginationResponseDto } =
await customModel.findAll({ ...pageable, modelName: 'community' }, qb); await customModel.findAll({ ...pageable, modelName: 'community' }, qb);
if (includeSpaces) {
baseResponseDto.data = baseResponseDto.data.map((community) => ({
...community,
spaces: this.spaceService.buildSpaceHierarchy(community.spaces || []),
}));
}
return new PageResponse<CommunityDto>( return new PageResponse<CommunityDto>(
baseResponseDto, baseResponseDto,
paginationResponseDto, paginationResponseDto,
); );
} catch (error) { } catch (error) {
// Generic error handling // Generic error handling
if (error instanceof HttpException) {
throw error;
}
throw new HttpException( throw new HttpException(
error.message || 'An error occurred while fetching communities.', error.message || 'An error occurred while fetching communities.',
HttpStatus.INTERNAL_SERVER_ERROR, HttpStatus.INTERNAL_SERVER_ERROR,

View File

@ -1,11 +1,11 @@
import { DeviceService } from '../services/device.service';
import { Controller, Get, Param, Query, UseGuards } from '@nestjs/common';
import { ApiTags, ApiBearerAuth, ApiOperation } from '@nestjs/swagger';
import { EnableDisableStatusEnum } from '@app/common/constants/days.enum';
import { ControllerRoute } from '@app/common/constants/controller-route'; import { ControllerRoute } from '@app/common/constants/controller-route';
import { PermissionsGuard } from 'src/guards/permissions.guard'; import { EnableDisableStatusEnum } from '@app/common/constants/days.enum';
import { Controller, Get, Param, Query, UseGuards } from '@nestjs/common';
import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
import { Permissions } from 'src/decorators/permissions.decorator'; import { Permissions } from 'src/decorators/permissions.decorator';
import { GetDoorLockDevices, ProjectParam } from '../dtos'; import { PermissionsGuard } from 'src/guards/permissions.guard';
import { GetDevicesFilterDto, ProjectParam } from '../dtos';
import { DeviceService } from '../services/device.service';
@ApiTags('Device Module') @ApiTags('Device Module')
@Controller({ @Controller({
@ -25,7 +25,7 @@ export class DeviceProjectController {
}) })
async getAllDevices( async getAllDevices(
@Param() param: ProjectParam, @Param() param: ProjectParam,
@Query() query: GetDoorLockDevices, @Query() query: GetDevicesFilterDto,
) { ) {
return await this.deviceService.getAllDevices(param, query); return await this.deviceService.getAllDevices(param, query);
} }

View File

@ -1,6 +1,7 @@
import { DeviceTypeEnum } from '@app/common/constants/device-type.enum'; import { DeviceTypeEnum } from '@app/common/constants/device-type.enum';
import { ApiProperty } from '@nestjs/swagger'; import { ApiProperty } from '@nestjs/swagger';
import { import {
IsArray,
IsEnum, IsEnum,
IsNotEmpty, IsNotEmpty,
IsOptional, IsOptional,
@ -41,16 +42,7 @@ export class GetDeviceLogsDto {
@IsOptional() @IsOptional()
public endTime: string; public endTime: string;
} }
export class GetDoorLockDevices {
@ApiProperty({
description: 'Device Type',
enum: DeviceTypeEnum,
required: false,
})
@IsEnum(DeviceTypeEnum)
@IsOptional()
public deviceType: DeviceTypeEnum;
}
export class GetDevicesBySpaceOrCommunityDto { export class GetDevicesBySpaceOrCommunityDto {
@ApiProperty({ @ApiProperty({
description: 'Device Product Type', description: 'Device Product Type',
@ -72,3 +64,23 @@ export class GetDevicesBySpaceOrCommunityDto {
@IsNotEmpty({ message: 'Either spaceUuid or communityUuid must be provided' }) @IsNotEmpty({ message: 'Either spaceUuid or communityUuid must be provided' })
requireEither?: never; // This ensures at least one of them is provided requireEither?: never; // This ensures at least one of them is provided
} }
export class GetDevicesFilterDto {
@ApiProperty({
description: 'Device Type',
enum: DeviceTypeEnum,
required: false,
})
@IsEnum(DeviceTypeEnum)
@IsOptional()
public deviceType: DeviceTypeEnum;
@ApiProperty({
description: 'List of Space IDs to filter devices',
required: false,
example: ['60d21b4667d0d8992e610c85', '60d21b4967d0d8992e610c86'],
})
@IsOptional()
@IsArray()
@IsUUID('4', { each: true })
public spaces?: string[];
}

View File

@ -53,7 +53,7 @@ import { DeviceSceneParamDto } from '../dtos/device.param.dto';
import { import {
GetDeviceLogsDto, GetDeviceLogsDto,
GetDevicesBySpaceOrCommunityDto, GetDevicesBySpaceOrCommunityDto,
GetDoorLockDevices, GetDevicesFilterDto,
} from '../dtos/get.device.dto'; } from '../dtos/get.device.dto';
import { import {
controlDeviceInterface, controlDeviceInterface,
@ -955,19 +955,20 @@ export class DeviceService {
async getAllDevices( async getAllDevices(
param: ProjectParam, param: ProjectParam,
query: GetDoorLockDevices, { deviceType, spaces }: GetDevicesFilterDto,
): Promise<BaseResponseDto> { ): Promise<BaseResponseDto> {
try { try {
await this.validateProject(param.projectUuid); await this.validateProject(param.projectUuid);
if (query.deviceType === DeviceTypeEnum.DOOR_LOCK) { if (deviceType === DeviceTypeEnum.DOOR_LOCK) {
return await this.getDoorLockDevices(param.projectUuid); return await this.getDoorLockDevices(param.projectUuid, spaces);
} else if (!query.deviceType) { } else if (!deviceType) {
const devices = await this.deviceRepository.find({ const devices = await this.deviceRepository.find({
where: { where: {
isActive: true, isActive: true,
spaceDevice: { spaceDevice: {
community: { project: { uuid: param.projectUuid } }, uuid: spaces && spaces.length ? In(spaces) : undefined,
spaceName: Not(ORPHAN_SPACE_NAME), spaceName: Not(ORPHAN_SPACE_NAME),
community: { project: { uuid: param.projectUuid } },
}, },
}, },
relations: [ relations: [
@ -1563,7 +1564,7 @@ export class DeviceService {
} }
} }
async getDoorLockDevices(projectUuid: string) { async getDoorLockDevices(projectUuid: string, spaces?: string[]) {
await this.validateProject(projectUuid); await this.validateProject(projectUuid);
const devices = await this.deviceRepository.find({ const devices = await this.deviceRepository.find({
@ -1573,6 +1574,7 @@ export class DeviceService {
}, },
spaceDevice: { spaceDevice: {
spaceName: Not(ORPHAN_SPACE_NAME), spaceName: Not(ORPHAN_SPACE_NAME),
uuid: spaces && spaces.length ? In(spaces) : undefined,
community: { community: {
project: { project: {
uuid: projectUuid, uuid: projectUuid,

View File

@ -26,8 +26,6 @@ async function bootstrap() {
rateLimit({ rateLimit({
windowMs: 5 * 60 * 1000, windowMs: 5 * 60 * 1000,
max: 500, max: 500,
standardHeaders: true,
legacyHeaders: false,
}), }),
); );
@ -36,7 +34,7 @@ async function bootstrap() {
next(); next();
}); });
app.getHttpAdapter().getInstance().set('trust proxy', 1); // app.getHttpAdapter().getInstance().set('trust proxy', 1);
app.use( app.use(
helmet({ helmet({

View File

@ -681,7 +681,7 @@ export class SpaceService {
} }
} }
private buildSpaceHierarchy(spaces: SpaceEntity[]): SpaceEntity[] { buildSpaceHierarchy(spaces: SpaceEntity[]): SpaceEntity[] {
const map = new Map<string, SpaceEntity>(); const map = new Map<string, SpaceEntity>();
// Step 1: Create a map of spaces by UUID // Step 1: Create a map of spaces by UUID