From ab4f5dee8f0522de81f374eb4738bf79499e0c32 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sun, 17 Mar 2024 10:57:12 +0300 Subject: [PATCH] Refactor approach to calculate alert values --- .../modules/eudr-alerts/alerts.repository.ts | 23 +- .../dashboard/eudr-dashboard.service.ts | 484 ++++++++---------- .../eudr-alerts/dto/alerts-output.dto.ts | 2 +- .../modules/eudr-alerts/dto/get-alerts.dto.ts | 12 + .../eudr-alerts/eudr.repositoty.interface.ts | 11 + api/test/utils/service-mocks.ts | 9 + 6 files changed, 256 insertions(+), 285 deletions(-) diff --git a/api/src/modules/eudr-alerts/alerts.repository.ts b/api/src/modules/eudr-alerts/alerts.repository.ts index abace2a2e..c7e3f92ad 100644 --- a/api/src/modules/eudr-alerts/alerts.repository.ts +++ b/api/src/modules/eudr-alerts/alerts.repository.ts @@ -12,6 +12,7 @@ import { import { DataSource } from 'typeorm'; import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto'; import { + AlertedGeoregionsBySupplier, EUDRAlertDatabaseResult, EUDRAlertDates, IEUDRAlertsRepository, @@ -57,12 +58,24 @@ export class AlertsRepository implements IEUDRAlertsRepository { this.createQueryBuilder(dto); // TODO: Make field selection dynamic queryBuilder.from(this.baseDataset, 'alerts'); - queryBuilder.select('alertdate', 'alertDate'); - queryBuilder.addSelect('alertconfidence', 'alertConfidence'); - queryBuilder.addSelect('year', 'alertYear'); - queryBuilder.addSelect('alertcount', 'alertCount'); + queryBuilder.select('alert_date', 'alertDate'); + queryBuilder.addSelect('supplierid', 'supplierId'); + queryBuilder.addSelect('alert_count', 'alertCount'); queryBuilder.addSelect('georegionid', 'geoRegionId'); - queryBuilder.orderBy('alertdate', 'ASC'); + queryBuilder.orderBy('alert_date', 'ASC'); + return this.query(queryBuilder); + } + + async getAlertedGeoRegionsBySupplier(dto: { + supplierIds: string[]; + startAlertDate: Date; + endAlertDate: Date; + }): Promise { + const queryBuilder: BigQueryAlertsQueryBuilder = + this.createQueryBuilder(dto); + queryBuilder.from(this.baseDataset, 'alerts'); + queryBuilder.select('georegionid', 'geoRegionId'); + queryBuilder.addSelect('supplierid', 'supplierId'); return this.query(queryBuilder); } diff --git a/api/src/modules/eudr-alerts/dashboard/eudr-dashboard.service.ts b/api/src/modules/eudr-alerts/dashboard/eudr-dashboard.service.ts index 39726e562..2b99a56d5 100644 --- a/api/src/modules/eudr-alerts/dashboard/eudr-dashboard.service.ts +++ b/api/src/modules/eudr-alerts/dashboard/eudr-dashboard.service.ts @@ -1,7 +1,7 @@ import { Inject, Injectable, NotFoundException } from '@nestjs/common'; import { DataSource, EntityManager, SelectQueryBuilder } from 'typeorm'; import { - EUDRAlertDatabaseResult, + AlertedGeoregionsBySupplier, IEUDRAlertsRepository, } from 'modules/eudr-alerts/eudr.repositoty.interface'; import { SourcingLocation } from 'modules/sourcing-locations/sourcing-location.entity'; @@ -50,26 +50,7 @@ export class EudrDashboardService { ); } - const entityMetadata: EntityMetadata[] = await this.getEntityMetadata(dto); - if (!entityMetadata.length) { - throw new NotFoundException( - 'Could not retrieve EUDR Data. Please contact the administrator', - ); - } - - const alertSummary: EUDRAlertDatabaseResult[] = - await this.eudrRepository.getAlertSummary({ - alertStartDate: dto.startAlertDate, - alertEnDate: dto.endAlertDate, - supplierIds: entityMetadata.map( - (entity: EntityMetadata) => entity.supplierId, - ), - }); - - const materials: Record< - EUDRDashBoardFields, - { totalPercentage: number; detail: any[] } - > = { + const materials: Record = { [EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS]: { totalPercentage: 0, detail: [], @@ -84,10 +65,7 @@ export class EudrDashboardService { }, }; - const origins: Record< - EUDRDashBoardFields, - { totalPercentage: number; detail: any[] } - > = { + const origins: Record = { [EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS]: { totalPercentage: 0, detail: [], @@ -101,302 +79,250 @@ export class EudrDashboardService { detail: [], }, }; - const alertMap: { - [key: string]: { - supplierId: string; - dfs: number; - tpl: number; - sda: number; - }; - } = alertSummary.reduce((acc: any, cur: EUDRAlertDatabaseResult) => { - acc[cur.supplierid] = { - supplierid: cur.supplierid, - dfs: cur.dfs, - sda: cur.sda, - }; - return acc; - }, {}); - - const entityMap: { - [key: string]: { - supplierId: string; - supplierName: string; - companyId: string; - materialId: string; - materialName: string; - adminRegionId: string; - adminRegionName: string; - totalBaselineVolume: number; - knownGeoRegions: number; - totalSourcingLocations: number; - isoA3: string; - }; - } = entityMetadata.reduce((acc: any, cur: EntityMetadata) => { - acc[cur.supplierId] = { ...cur }; - return acc; - }, {}); - - const supplierToMaterials: Map = - new Map(); - const supplierToOriginis: Map< - string, - { name: string; id: string; isoA3: string }[] - > = new Map(); - const materialMap: Map< - string, - { - materialName: string; - suppliers: Set; - dfsSuppliers: number; - sdaSuppliers: number; - totalSourcingLocations: number; - knownGeoRegions: number; - } - > = new Map(); - const originMap: Map< - string, - { - originName: string; - suppliers: Set; - dfsSuppliers: number; - sdaSuppliers: number; - totalSourcingLocations: number; - knownGeoRegions: number; - isoA3: string; + + const entityMetadata: EntityMetadata[] = await this.getEntityMetadata(dto); + if (!entityMetadata.length) { + throw new NotFoundException( + 'Could not retrieve EUDR Data. Please contact the administrator', + ); + } + + const alerts: AlertedGeoregionsBySupplier[] = + await this.eudrRepository.getAlertedGeoRegionsBySupplier({ + startAlertDate: dto.startAlertDate, + endAlertDate: dto.endAlertDate, + supplierIds: entityMetadata.map( + (entity: EntityMetadata) => entity.supplierId, + ), + }); + + const alertMap = new Map>(); + + alerts.forEach((alert: AlertedGeoregionsBySupplier) => { + const { supplierId, geoRegionId } = alert; + + if (!alertMap.has(supplierId)) { + alertMap.set(supplierId, new Set()); } - > = new Map(); - entityMetadata.forEach((entity: EntityMetadata) => { + // @ts-ignore + alertMap.get(supplierId).add(geoRegionId); + }); + + const suppliersMap = new Map(); + const materialsMap = new Map(); + const originsMap = new Map(); + + entityMetadata.forEach((entity) => { const { - materialId, supplierId, + materialId, adminRegionId, + totalSourcingLocations, + knownGeoRegions, + supplierName, + companyId, materialName, adminRegionName, - knownGeoRegions, - totalSourcingLocations, + totalBaselineVolume, isoA3, } = entity; - const unknownGeoRegions: number = - totalSourcingLocations - knownGeoRegions; - const tplPercentage: number = - (unknownGeoRegions / totalSourcingLocations) * 100; - - if (alertMap[supplierId]) { - alertMap[supplierId].tpl = tplPercentage; - } else { - alertMap[supplierId] = { + const alertedGeoRegionsCount = alertMap.get(supplierId)?.size || 0; + const nonAlertedGeoRegions = + parseInt(String(totalSourcingLocations)) - + parseInt(String(alertedGeoRegionsCount)); + const unknownGeoRegions = + parseInt(String(totalSourcingLocations)) - + parseInt(String(knownGeoRegions)); + + const sdaPercentage = + (alertedGeoRegionsCount / totalSourcingLocations) * 100; + const tplPercentage = (unknownGeoRegions / totalSourcingLocations) * 100; + const dfsPercentage = + 100 - (sdaPercentage + tplPercentage) > 0 + ? 100 - (sdaPercentage + tplPercentage) + : 0; + + if (!suppliersMap.has(supplierId)) { + suppliersMap.set(supplierId, { supplierId, + supplierName, + companyId, + materials: [], + origins: [], + totalBaselineVolume: 0, dfs: 0, sda: 0, - tpl: tplPercentage, - }; - } - - if (!supplierToMaterials.has(supplierId)) { - supplierToMaterials.set(supplierId, []); - } - if (!supplierToOriginis.has(supplierId)) { - supplierToOriginis.set(supplierId, []); + tpl: 0, + }); } + const supplier = suppliersMap.get(supplierId); + supplier.totalBaselineVolume = totalBaselineVolume; + supplier.dfs = dfsPercentage; + supplier.sda = sdaPercentage; + supplier.tpl = tplPercentage; + supplier.materials.push({ id: materialId, name: materialName }); + supplier.origins.push({ + id: adminRegionId, + name: adminRegionName, + isoA3: isoA3, + }); - if (!materialMap.has(materialId)) { - materialMap.set(materialId, { + if (!materialsMap.has(materialId)) { + materialsMap.set(materialId, { + materialId, materialName, suppliers: new Set(), - dfsSuppliers: 0, - sdaSuppliers: 0, totalSourcingLocations: 0, knownGeoRegions: 0, + alertedGeoRegions: 0, }); } - if (!originMap.has(adminRegionId)) { - originMap.set(adminRegionId, { - originName: adminRegionName, - isoA3: isoA3, + const material = materialsMap.get(materialId); + material.suppliers.add(supplierId); + material.totalSourcingLocations += parseFloat( + String(totalSourcingLocations), + ); + material.knownGeoRegions += parseInt(String(knownGeoRegions)); + material.alertedGeoRegions += alertMap.get(supplierId)?.size || 0; + + if (!originsMap.has(adminRegionId)) { + originsMap.set(adminRegionId, { + adminRegionId, + adminRegionName, + isoA3, suppliers: new Set(), - dfsSuppliers: 0, - sdaSuppliers: 0, totalSourcingLocations: 0, knownGeoRegions: 0, + alertedGeoRegions: 0, }); } - - const material: any = materialMap.get(materialId); - const origin: any = originMap.get(adminRegionId); - material.suppliers.add(supplierId); - material.totalSourcingLocations += totalSourcingLocations; - material.knownGeoRegions += knownGeoRegions; + const origin = originsMap.get(adminRegionId); origin.suppliers.add(supplierId); - origin.totalSourcingLocations += totalSourcingLocations; - origin.knownGeoRegions += knownGeoRegions; - - const alertData: any = alertMap[supplierId]; - if (alertData) { - if (alertData.dfs > 0) { - material.dfsSuppliers += 1; - origin.dfsSuppliers += 1; - } - if (alertData.sda > 0) { - material.sdaSuppliers += 1; - origin.sdaSuppliers += 1; - } - } + origin.totalSourcingLocations += parseInt(String(totalSourcingLocations)); + origin.knownGeoRegions += parseInt(String(knownGeoRegions)); + origin.alertedGeoRegions += alertMap.get(supplierId)?.size || 0; + }); - // Añadir detalles de material y región al supplier - supplierToMaterials.get(supplierId)!.push({ + materialsMap.forEach((material, materialId) => { + const { + materialName, + totalSourcingLocations, + knownGeoRegions, + alertedGeoRegions, + } = material; + const nonAlertedGeoRegions = knownGeoRegions - alertedGeoRegions; + const unknownGeoRegions = totalSourcingLocations - knownGeoRegions; + + const sdaPercentage = (alertedGeoRegions / totalSourcingLocations) * 100; + const tplPercentage = (unknownGeoRegions / totalSourcingLocations) * 100; + const dfsPercentage = + 100 - (sdaPercentage + tplPercentage) > 0 + ? 100 - (sdaPercentage + tplPercentage) + : 0; + + materials[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.push({ name: materialName, id: materialId, + value: dfsPercentage, }); - supplierToOriginis.get(supplierId)!.push({ - name: adminRegionName, - id: adminRegionId, - isoA3: isoA3, + materials[ + EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS + ].detail.push({ + name: materialName, + id: materialId, + value: sdaPercentage, }); - }); - - materialMap.forEach( - ( - { - materialName, - suppliers, - totalSourcingLocations, - knownGeoRegions, - dfsSuppliers, - sdaSuppliers, - }, - materialId: string, - ) => { - const tplPercentage: number = - ((totalSourcingLocations - knownGeoRegions) / suppliers.size) * 100; - materials['Suppliers with no location data'].detail.push({ - name: materialName, - value: tplPercentage, - }); - const dfsPercentage: number = (dfsSuppliers / suppliers.size) * 100; - materials['Deforestation-free suppliers'].detail.push({ - name: materialName, - value: dfsPercentage, - }); - const sdaPercentage: number = (sdaSuppliers / suppliers.size) * 100; - materials['Suppliers with deforestation alerts'].detail.push({ - name: materialName, - value: sdaPercentage, - }); - }, - ); - - originMap.forEach( - ( - { - originName, - suppliers, - totalSourcingLocations, - knownGeoRegions, - dfsSuppliers, - sdaSuppliers, - isoA3, - }, - adminRegionId: string, - ) => { - const tplPercentage: number = - ((totalSourcingLocations - knownGeoRegions) / suppliers.size) * 100; - origins[ - EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA - ].detail.push({ - name: originName, - value: tplPercentage, - isoA3, - }); - const dfsPercentage: number = (dfsSuppliers / suppliers.size) * 100; - origins[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.push({ - name: originName, - value: dfsPercentage, - isoA3, - }); - const sdaPercentage: number = (sdaSuppliers / suppliers.size) * 100; - origins[ - EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS - ].detail.push({ - name: originName, - value: sdaPercentage, - isoA3, - }); - }, - ); - - materials[ - EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA - ].totalPercentage = materials[ EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA - ].detail.reduce((acc: number, cur: any) => acc + cur.value, 0) / - materials[EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA].detail - .length; - - materials[ - EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS - ].totalPercentage = - materials[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.reduce( - (acc: number, cur: any) => acc + cur.value, - 0, - ) / - materials[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.length; + ].detail.push({ + name: materialName, + id: materialId, + value: tplPercentage, + }); + }); - materials[ - EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS - ].totalPercentage = - materials[ - EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS - ].detail.reduce((acc: number, cur: any) => acc + cur.value, 0) / - materials[EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS].detail - .length; + // @ts-ignore + Object.keys(materials).forEach((key: EUDRDashBoardFields) => { + const totalPercentage: number = + materials[key].detail.reduce( + (acc: number, cur: any) => acc + cur.value, + 0, + ) / materials[key].detail.length; + materials[key].totalPercentage = totalPercentage; + }); - origins[ - EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA - ].totalPercentage = + originsMap.forEach((origin, adminRegionId) => { + const { + adminRegionName, + isoA3, + totalSourcingLocations, + knownGeoRegions, + alertedGeoRegions, + } = origin; + const nonAlertedGeoRegions = knownGeoRegions - alertedGeoRegions; + const unknownGeoRegions = totalSourcingLocations - knownGeoRegions; + + const sdaPercentage = (alertedGeoRegions / totalSourcingLocations) * 100; + const tplPercentage = (unknownGeoRegions / totalSourcingLocations) * 100; + const dfsPercentage = + 100 - (sdaPercentage + tplPercentage) > 0 + ? 100 - (sdaPercentage + tplPercentage) + : 0; + + origins[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.push({ + name: adminRegionName, + id: adminRegionId, + isoA3: isoA3, + value: dfsPercentage, + }); origins[ - EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA - ].detail.reduce((acc: number, cur: any) => acc + cur.value, 0) / - origins[EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA].detail - .length; + EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS + ].detail.push({ + name: adminRegionName, + id: adminRegionId, + isoA3: isoA3, + value: sdaPercentage, + }); + origins[EUDRDashBoardFields.SUPPLIERS_WITH_NO_LOCATION_DATA].detail.push({ + name: adminRegionName, + id: adminRegionId, + isoA3: isoA3, + value: tplPercentage, + }); + }); - origins[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].totalPercentage = - origins[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.reduce( - (acc: number, cur: any) => acc + cur.value, - 0, - ) / - origins[EUDRDashBoardFields.DEFORASTATION_FREE_SUPPLIERS].detail.length; + // @ts-ignore + Object.keys(origins).forEach((key: EUDRDashBoardFields) => { + const totalPercentage: number = + origins[key].detail.reduce( + (acc: number, cur: any) => acc + cur.value, + 0, + ) / origins[key].detail.length; + origins[key].totalPercentage = isNaN(totalPercentage) + ? 0 + : totalPercentage; + }); - origins[ - EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS - ].totalPercentage = - origins[ - EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS - ].detail.reduce((acc: number, cur: any) => acc + cur.value, 0) / - origins[EUDRDashBoardFields.SUPPLIERS_WITH_DEFORASTATION_ALERTS].detail - .length; - - const table: DashBoardTableElements[] = Object.keys(alertMap).map( - (key: string) => { - return { - supplierId: key, - supplierName: entityMap[key].supplierName, - companyId: entityMap[key].companyId, - baselineVolume: entityMap[key].totalBaselineVolume, - dfs: alertMap[key].dfs, - sda: alertMap[key].sda, - tpl: alertMap[key].tpl, - materials: supplierToMaterials.get(key) || [], - origins: supplierToOriginis.get(key) || [], - }; - }, - ); + const table: DashBoardTableElements[] = []; + suppliersMap.forEach((supplier: any) => { + table.push({ + supplierId: supplier.supplierId, + supplierName: supplier.supplierName, + companyId: supplier.companyId, + materials: supplier.materials, + origins: supplier.origins, + baselineVolume: supplier.totalBaselineVolume, + dfs: supplier.dfs, + sda: supplier.sda, + tpl: supplier.tpl, + }); + }); return { table: table, - breakDown: { materials, origins } as EUDRBreakDown, + breakDown: { materials, origins } as any, }; } diff --git a/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts b/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts index d61ab5583..3167d2468 100644 --- a/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts +++ b/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts @@ -4,7 +4,7 @@ export type AlertsOutput = { value: Date | string; }; year: number; - alertConfidence: 'low' | 'medium' | 'high' | 'very high'; + supplierId: string; geoRegionId: string; }; diff --git a/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts b/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts index 0f55caac4..df21cbafd 100644 --- a/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts +++ b/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts @@ -50,6 +50,18 @@ export class GetEUDRAlertsDto extends GetEUDRAlertDatesDto { alertConfidence?: 'high' | 'medium' | 'low'; + @ApiPropertyOptional() + @IsOptional() + @IsDate() + @Type(() => Date) + startAlertDate?: Date; + + @ApiPropertyOptional() + @IsOptional() + @IsDate() + @Type(() => Date) + endAlertDate?: Date; + @ApiPropertyOptional() @IsOptional() @IsInt() diff --git a/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts b/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts index 06da6c8bd..a35792fa3 100644 --- a/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts +++ b/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts @@ -18,6 +18,11 @@ export interface EUDRAlertDatabaseResult { sda: number; } +export interface AlertedGeoregionsBySupplier { + supplierId: string; + geoRegionId: string; +} + export type GetAlertSummary = { alertStartDate?: Date; alertEnDate?: Date; @@ -31,4 +36,10 @@ export interface IEUDRAlertsRepository { getDates(dto: GetEUDRAlertsDto): Promise; getAlertSummary(dto: GetAlertSummary): Promise; + + getAlertedGeoRegionsBySupplier(dto: { + supplierIds: string[]; + startAlertDate: Date; + endAlertDate: Date; + }): Promise; } diff --git a/api/test/utils/service-mocks.ts b/api/test/utils/service-mocks.ts index 3df85a9fb..1dfaeae4c 100644 --- a/api/test/utils/service-mocks.ts +++ b/api/test/utils/service-mocks.ts @@ -4,6 +4,7 @@ import { } from '../../src/modules/notifications/email/email.service.interface'; import { Logger } from '@nestjs/common'; import { + AlertedGeoregionsBySupplier, EUDRAlertDatabaseResult, EUDRAlertDates, GetAlertSummary, @@ -38,4 +39,12 @@ export class MockAlertRepository implements IEUDRAlertsRepository { getAlertSummary(dto: GetAlertSummary): Promise { return Promise.resolve([]); } + + getAlertedGeoRegionsBySupplier(dto: { + supplierIds: string[]; + startAlertDate: Date; + endAlertDate: Date; + }): Promise { + return Promise.resolve([]); + } }