From 0659af0721e424808b6fb9f1158c863bf4381f1c Mon Sep 17 00:00:00 2001 From: KevSanchez Date: Fri, 2 Sep 2022 10:54:45 +0200 Subject: [PATCH] chore(ImpactCalculator): Polished new service to calculate raw values for all sourcing records The prototpye for for calculating all raw values for all sourcing records has revamped/polished to final status, with approval pending once tested. Also the the feature flag selection has been moved to sourcing data import along with a bit of refactoring. --- .../sourcing-data-import.service.ts | 6 +- .../indicator-records.service.ts | 56 +- .../services/impact-calculator.service.ts | 620 ++++++++++++------ ...ing-records-with-indicator-raw-data.dto.ts | 5 + .../indicator-records.service.spec.ts | 6 +- 5 files changed, 476 insertions(+), 217 deletions(-) diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index d4ce8996db..c906d2b04d 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -28,6 +28,7 @@ import { GeoCodingAbstractClass } from 'modules/geo-coding/geo-coding-abstract-c import { MissingH3DataError } from 'modules/indicator-records/errors/missing-h3-data.error'; import { TasksService } from 'modules/tasks/tasks.service'; import { IndicatorRecord } from 'modules/indicator-records/indicator-record.entity'; +import * as config from 'config'; export interface LocationData { locationAddressInput?: string; @@ -139,7 +140,10 @@ export class SourcingDataImportService { // Getting H3 data for calculations is done within DB so we need to improve the error handling // TBD: What to do when there is no H3 for a Material try { - await this.indicatorRecordsService.createIndicatorRecordsForAllSourcingRecords(); + // TODO remove feature flag selection, once the solution has been approved + config.get('featureFlags.simpleImportCalculations') + ? await this.indicatorRecordsService.createIndicatorRecordsForAllSourcingRecordsV2() + : await this.indicatorRecordsService.createIndicatorRecordsForAllSourcingRecords(); this.logger.log('Indicator Records generated'); // TODO: Hack to force m.view refresh once Indicator Records are persisted. This should be automagically // done by the AfterInser() event listener placed in indicator-record.entity.ts diff --git a/api/src/modules/indicator-records/indicator-records.service.ts b/api/src/modules/indicator-records/indicator-records.service.ts index 26169cad7f..dd7beae988 100644 --- a/api/src/modules/indicator-records/indicator-records.service.ts +++ b/api/src/modules/indicator-records/indicator-records.service.ts @@ -166,11 +166,8 @@ export class IndicatorRecordsService extends AppBaseService< INDICATOR_TYPES.BIODIVERSITY_LOSS, ]; - const rawData: SourcingRecordsWithIndicatorRawDataDto[] = config.get( - 'featureFlags.simpleImportCalculations', - ) - ? await this.impactCalculatorService.calculateAllSourcingRecords() - : await this.indicatorRecordRepository.getIndicatorRawDataForAllSourcingRecords(); + const rawData: SourcingRecordsWithIndicatorRawDataDto[] = + await this.indicatorRecordRepository.getIndicatorRawDataForAllSourcingRecords(); const calculatedData2: IndicatorRecordCalculatedValuesDto[] = rawData.map( (sourcingRecordData: SourcingRecordsWithIndicatorRawDataDto) => { @@ -226,6 +223,55 @@ export class IndicatorRecordsService extends AppBaseService< await this.indicatorRecordRepository.saveChunks(indicatorRecords); } + async createIndicatorRecordsForAllSourcingRecordsV2(): Promise { + //Calculate raw impact Data for all available indicators on the system + const indicators: Indicator[] = + await this.indicatorService.getAllIndicators(); + + const rawData: SourcingRecordsWithIndicatorRawDataDto[] = + await this.impactCalculatorService.calculateAllRawValuesForAllSourcingRecords( + indicators, + ); + + const calculatedData: IndicatorRecordCalculatedValuesDto[] = rawData.map( + (sourcingRecordData: SourcingRecordsWithIndicatorRawDataDto) => { + // Small DTO transformation for calculation method + const indicatorComputedRawDataDto: IndicatorComputedRawDataDto = { + harvestedArea: sourcingRecordData.harvestedArea, + production: sourcingRecordData.production, + indicatorValues: sourcingRecordData.indicatorValues, + }; + + return this.calculateIndicatorValues( + sourcingRecordData.sourcingRecordId, + sourcingRecordData.tonnage, + sourcingRecordData.materialH3DataId, + indicatorComputedRawDataDto, + ); + }, + ); + + // Create IndicatorRecord instances + const indicatorRecords: IndicatorRecord[] = []; + for (const calculatedIndicatorRecords of calculatedData) { + for (const indicator of indicators) { + indicatorRecords.push( + IndicatorRecord.merge(new IndicatorRecord(), { + value: calculatedIndicatorRecords.values.get( + indicator.nameCode as INDICATOR_TYPES, + ), + indicatorId: indicator.id, + status: INDICATOR_RECORD_STATUS.SUCCESS, + sourcingRecordId: calculatedIndicatorRecords.sourcingRecordId, + scaler: calculatedIndicatorRecords.production, + materialH3DataId: calculatedIndicatorRecords.materialH3DataId, + }), + ); + } + } + await this.indicatorRecordRepository.saveChunks(indicatorRecords); + } + /** * @description Creates Indicator-Records for a single Sourcing-Record, by first retrieving Raw Indicator data from the DB, then applying * the methodology and persist new Indicator Records diff --git a/api/src/modules/indicator-records/services/impact-calculator.service.ts b/api/src/modules/indicator-records/services/impact-calculator.service.ts index 7c5630b223..340056fd44 100644 --- a/api/src/modules/indicator-records/services/impact-calculator.service.ts +++ b/api/src/modules/indicator-records/services/impact-calculator.service.ts @@ -4,62 +4,241 @@ import { QueryRunner, SelectQueryBuilder, } from 'typeorm'; -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; import { MATERIAL_TO_H3_TYPE } from 'modules/materials/material-to-h3.entity'; import { H3Data } from 'modules/h3-data/h3-data.entity'; -import { INDICATOR_TYPES } from 'modules/indicators/indicator.entity'; +import { + Indicator, + INDICATOR_TYPES, +} from 'modules/indicators/indicator.entity'; import { SourcingRecordsWithIndicatorRawDataDto } from 'modules/sourcing-records/dto/sourcing-records-with-indicator-raw-data.dto'; +type MaterialSelectStatementGenerator = ( + materialsH3s: Map, +) => string; + +type IndicatorSelectStatementGenerator = ( + materialsH3s: Map, + indicatorH3s: Map, +) => string; + @Injectable() export class ImpactCalculatorService { logger: Logger = new Logger(this.constructor.name); + private readonly materialSelectStatmenteGenerator: Record< + MATERIAL_TO_H3_TYPE, + MaterialSelectStatementGenerator + > = { + [MATERIAL_TO_H3_TYPE.HARVEST]: harvestMaterialSelectStatementGenerator, + [MATERIAL_TO_H3_TYPE.PRODUCER]: producerMaterialSelectStatementGenerator, + }; + private readonly indicatorSelectStatementGenerator: Record< + INDICATOR_TYPES, + IndicatorSelectStatementGenerator + > = { + [INDICATOR_TYPES.BIODIVERSITY_LOSS]: bioDiversitySelectStatementGenerator, + [INDICATOR_TYPES.CARBON_EMISSIONS]: carbonSelectStatementGenerator, + [INDICATOR_TYPES.DEFORESTATION]: deforestationSelectStatementGenerator, + [INDICATOR_TYPES.UNSUSTAINABLE_WATER_USE]: waterSelectStatementGenerator, + }; + + // STEPS + /** + * 1. Get material h3 data table and column name + * 1.1 For each year, get the closest available material h3 data + * 2. Get indicator h3 data table and column name + * 2.1 For each year, get the closest available material h3 data + ** Look at how interventions impact calculus implements this. + * + * 2.1 Get deforestation indicator h3 data and column name (because this indicator needs to be crossed with this data) + * + * CRAZY IDEAZ: + * 1. We have 12 years to calculate impact: 2010-2022 (12 DB calls) + * 2. We have 3 available years to calculate impact: 2010, 2014, 2020 + * + * Before performing any call, can we determine that Sourcing Records from 2010 to 2012 will use data of 2010 + * from 2013 to 2017 will use data of 2014 + * from 2018 to 2022 will use data of 2022 + * + * Knowing this, can we calculate impacts for those years simultaneosly (arent we doing that now anyway?) in 3 DB CALLS + * instead of doing 12, each for one year? + * + * LONG STORY SHORT: + * + * Can we do as much calls as different h3 data tables we need to attack (in this case 3) + * instead of doing as much calls as years we have to calculate impact for(in this case 12) + * + *There's another problem; every indicator/material might not have data available for the same years, an indicator having + * data for 2010 and 2020, and another indicator for 2012 and 2017 + * seems like the root of the problem might be pretty early in the process, when deciding what (or more likely when) data to use + * for calculations + * what are the possible strategies to calculate the gap years? shgould it be configurable AFTER deployment? + * closest? that might be resolved by copying columns on the H3 info table on the H3 import + * mean between the closest ones? that's more difficult, might be possible in the H3 import? but it would be something + * not configurable once deployed + */ + + /** + * Calculates raw values for all indicators available in the system and all types of material, + * from all SourcingRecords in the DB that are not part of an intervention. + * - First, it gets all Sourcing Records, plus related data (its sourcing location, material) + * - Then for each sourcing record, it gets all the closest H3 Data to its years, and then + * calculates the material/indicator raw values in a single DB call + * @param indicators + */ + async calculateAllRawValuesForAllSourcingRecords( + indicators: Indicator[], + ): Promise { + const connection: Connection = getConnection(); + const queryRunner: QueryRunner = connection.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + + this.logger.log( + 'Calculating All Raw Impact/Material Values for all SourcingRecords', + ); + + let result: SourcingRecordsWithIndicatorRawDataDto[] = []; + try { + const sourcingRecords: any[] = await this.getAllSourcingRecordsData( + connection, + queryRunner, + ); + + //Generate SourcingRecordsWithIndicatorRawDataDtos from each sourcing record's + //georegion, material and year, for the given indicators + result = await Promise.all( + sourcingRecords.map(async (sourcingRecord: any) => { + const rawValues: any = + await this.calculateAllRawValuesForGeoRegionAndYear( + connection, + queryRunner, + indicators, + sourcingRecord.geoRegionId, + sourcingRecord.materialId, + sourcingRecord.year, + 5, // Max resolution + ); + + return this.createResultInstance( + sourcingRecord, + rawValues, + indicators, + ); + }), + ); + + await queryRunner.commitTransaction(); + } catch (err) { + // rollback changes before throwing error + await queryRunner.rollbackTransaction(); + this.logger.error(err); + } finally { + // release query runner which is manually created + await queryRunner.release(); + } + + if (!result.length) { + throw new Error( + 'No raw data could be calculated could be calculated for all sourcing records', + ); + } - async calculateImpact( + return result; + } + + private async getAllSourcingRecordsData( + connection: Connection, + queryRunner: QueryRunner, + ): Promise { + const sourcingRecordsQuery: SelectQueryBuilder = connection + .createQueryBuilder() + .select([ + `sr.id as "sourcingRecordId", + sr.tonnage, + sr.year, + sl.id as "sourcingLocationId", + sl."materialId", + sl."geoRegionId", + mth."h3DataId" as "materialH3DataId"`, + ]) + .from('sourcing_records', 'sr') + .innerJoin('sourcing_location', 'sl', 'sl.id = sr."sourcingLocationId"') + .innerJoin( + (subQuery: SelectQueryBuilder) => { + return subQuery + .select('"materialId"') + .addSelect('"h3DataId"') + .from('material_to_h3', 'material_to_h3') + .where(`type='${MATERIAL_TO_H3_TYPE.HARVEST}'`); + }, + 'mth', + 'mth."materialId" = sl."materialId"', + ) + .where('sl."scenarioInterventionId" IS NULL') + .andWhere('sl."interventionType" IS NULL'); + + return queryRunner.query(sourcingRecordsQuery.getQuery()); + } + + private createResultInstance( + sourcingRecord: any, + rawValues: any, + indicators: Indicator[], + ): SourcingRecordsWithIndicatorRawDataDto { + const indicatorValues: Map = new Map(); + for (const indicator of indicators) { + const indicatorType: INDICATOR_TYPES = + indicator.nameCode as INDICATOR_TYPES; + + indicatorValues.set( + indicatorType, + rawValues[generateValueAlias(indicatorType)], + ); + } + + return { + sourcingRecordId: sourcingRecord.sourcingRecordId, + tonnage: sourcingRecord.tonnage, + year: sourcingRecord.year, + + sourcingLocationId: sourcingRecord.sourcingLocationId, + production: rawValues[generateValueAlias(MATERIAL_TO_H3_TYPE.PRODUCER)], + harvestedArea: rawValues[generateValueAlias(MATERIAL_TO_H3_TYPE.HARVEST)], + + // TODO remove this hardcoded fields once the "simpleImportCalculations" feature has been tested/approved + rawDeforestation: rawValues[`DF_LUC_T_value`], + rawBiodiversity: rawValues[`BL_LUC_T_value`], + rawCarbon: rawValues[`GHG_LUC_T_value`], + rawWater: rawValues[`UWU_T_value`], + + indicatorValues, + materialH3DataId: sourcingRecord.materialH3DataId, + }; + } + + /** + * Calculates all raw values for the given geoRegionId and year, with the H3 Datas of + * indicators and material types closest to the given year + * This means raw values for: + * - all types of the given MaterialId + * - all indicators in the array parameter + * @param connection + * @param queryRunner + * @param georegionId + * @param materialId + * @param year + * @param indicators + */ + async calculateAllRawValuesForGeoRegionAndYear( connection: Connection, queryRunner: QueryRunner, + indicators: Indicator[], georegionId: string, materialId: string, year: number, + resolution?: number, ): Promise { - // STEPS - /** - * 1. Get material h3 data table and column name - * 1.1 For each year, get the closest available material h3 data - * 2. Get indicator h3 data table and column name - * 2.1 For each year, get the closest available material h3 data - ** Look at how interventions impact calculus implements this. - * - * 2.1 Get deforestation indicator h3 data and column name (because this indicator needs to be crossed with this data) - * - * CRAZY IDEAZ: - * 1. We have 12 years to calculate impact: 2010-2022 (12 DB calls) - * 2. We have 3 available years to calculate impact: 2010, 2014, 2020 - * - * Before performing any call, can we determine that Sourcing Records from 2010 to 2012 will use data of 2010 - * from 2013 to 2017 will use data of 2014 - * from 2018 to 2022 will use data of 2022 - * - * Knowing this, can we calculate impacts for those years simultaneosly (arent we doing that now anyway?) in 3 DB CALLS - * instead of doing 12, each for one year? - * - * LONG STORY SHORT: - * - * Can we do as much calls as different h3 data tables we need to attack (in this case 3) - * instead of doing as much calls as years we have to calculate impact for(in this case 12) - * - *There's another problem; every indicator/material might not have data available for the same years, an indicator having - * data for 2010 and 2020, and another indicator for 2012 and 2017 - * seems like the root of the problem might be pretty early in the process, when deciding what (or more likely when) data to use - * for calculations - * what are the possible strategies to calculate the gap years? shgould it be configurable AFTER deployment? - * closest? that might be resolved by copying columns on the H3 info table on the H3 import - * mean between the closest ones? that's more difficult, might be possible in the H3 import? but it would be something - * not configurable once deployed - * - - * - */ - const materialH3s: Map = await this.getAllMaterialH3sByClosestYear( connection, @@ -67,179 +246,74 @@ export class ImpactCalculatorService { materialId, year, ); + const indicatorTypes: INDICATOR_TYPES[] = indicators.map( + (value: Indicator) => value.nameCode as INDICATOR_TYPES, + ); const indicatorH3s: Map = await this.getIndicatorH3sByTypeAndClosestYear( connection, queryRunner, - Object.values(INDICATOR_TYPES), + indicatorTypes, year, ); - const producerH3: H3Data = materialH3s.get(MATERIAL_TO_H3_TYPE.PRODUCER)!; - const harvestH3: H3Data = materialH3s.get(MATERIAL_TO_H3_TYPE.HARVEST)!; - const bioH3: H3Data = indicatorH3s.get(INDICATOR_TYPES.BIODIVERSITY_LOSS)!; - const deforestH3: H3Data = indicatorH3s.get(INDICATOR_TYPES.DEFORESTATION)!; - const carbonH3: H3Data = indicatorH3s.get( - INDICATOR_TYPES.CARBON_EMISSIONS, - )!; - const waterH3: H3Data = indicatorH3s.get( - INDICATOR_TYPES.UNSUSTAINABLE_WATER_USE, - )!; + //Use the expanded list of H3 indexes corresponding to the geoRegion id + //as the base table for the query. Since everything will be joined by h3 indexes, + // it is assumed that all material/indicator H3 tables have same h3 indexes at the max resolution + // available (even tho const values: SelectQueryBuilder = await connection .createQueryBuilder() - - //SELECT aggregation statements - .select(`sum( "harvestH3"."${harvestH3.h3columnName}" )`, 'harvestedArea') - .addSelect( - `sum( "producerH3"."${producerH3.h3columnName}" )`, - 'production', - ) - .addSelect( - `sum( "harvestH3"."${harvestH3.h3columnName}" * "deforestH3"."${deforestH3.h3columnName}" ` + - `* "bioH3"."${bioH3.h3columnName}" * (1/0.0001) )`, - 'rawBiodiversity', - ) - .addSelect( - `sum( "harvestH3"."${harvestH3.h3columnName}" * "deforestH3"."${deforestH3.h3columnName}" ` + - `* "carbonH3"."${carbonH3.h3columnName}")`, - 'rawCarbon', - ) - .addSelect( - `sum( "harvestH3"."${harvestH3.h3columnName}" * "deforestH3"."${deforestH3.h3columnName}" )`, - 'rawDeforestation', - ) - .addSelect( - `sum( "waterH3"."${waterH3.h3columnName}" * 0.001)`, - 'rawWater', - ) - //FROM .from( - `(select * from get_h3_uncompact_geo_region('${georegionId}', 6))`, + `(select * from get_h3_uncompact_geo_region('${georegionId}', ${resolution}))`, 'geoRegion', - ) - .innerJoin( - producerH3.h3tableName, - 'producerH3', - `"producerH3".h3index = "geoRegion".h3index`, - ) - .innerJoin( - harvestH3.h3tableName, - 'harvestH3', - `"harvestH3".h3index = "geoRegion".h3index`, - ) - .innerJoin( - bioH3.h3tableName, - 'bioH3', - '"bioH3".h3index = "geoRegion".h3index', - ) - .innerJoin( - carbonH3.h3tableName, - 'carbonH3', - '"carbonH3".h3index = "geoRegion".h3index', - ) - .innerJoin( - deforestH3.h3tableName, - 'deforestH3', - '"deforestH3".h3index = "geoRegion".h3index', - ) - .innerJoin( - waterH3.h3tableName, - 'waterH3', - '"waterH3".h3index = "geoRegion".h3index', ); - const result: any = await queryRunner.query(values.getQuery()); - if (!result.length) - this.logger.warn( - `Could not retrieve Sourcing Records with weighted indicator values`, + //Material FROM and SELECT statements + for (const materialType of Object.values(MATERIAL_TO_H3_TYPE)) { + values.addSelect( + this.materialSelectStatmenteGenerator[materialType](materialH3s), + generateValueAlias(materialType), ); - return result[0]; - } - - async calculateAllSourcingRecords(): Promise< - SourcingRecordsWithIndicatorRawDataDto[] - > { - const connection: Connection = getConnection(); - const queryRunner: QueryRunner = connection.createQueryRunner(); - await queryRunner.connect(); - await queryRunner.startTransaction(); + } - let result: SourcingRecordsWithIndicatorRawDataDto[] = []; - try { - const sourcingRecordsQuery: SelectQueryBuilder = connection - .createQueryBuilder() - .select([ - `sr.id as "sourcingRecordId", - sr.tonnage, - sr.year, - sl.id as "sourcingLocationId", - sl."materialId", - sl."geoRegionId", - mth."h3DataId" as "materialH3DataId"`, - ]) - .from('sourcing_records', 'sr') - .innerJoin('sourcing_location', 'sl', 'sl.id = sr."sourcingLocationId"') - .innerJoin( - (subQuery: SelectQueryBuilder) => { - return subQuery - .select('"materialId"') - .addSelect('"h3DataId"') - .from('material_to_h3', 'material_to_h3') - .where(`type='${MATERIAL_TO_H3_TYPE.HARVEST}'`); - }, - 'mth', - 'mth."materialId" = sl."materialId"', - ) - .where('sl."scenarioInterventionId" IS NULL') - .andWhere('sl."interventionType" IS NULL'); - - const sourcingRecords: any[] = await queryRunner.query( - sourcingRecordsQuery.getQuery(), + for (const [materialType, materialH3Data] of materialH3s) { + values.innerJoin( + materialH3Data.h3tableName, + `${materialType}`, + `"${materialType}".h3index = "geoRegion".h3index`, ); - result = await Promise.all( - sourcingRecords.map(async (sourcingRecord: any) => { - const rawValues: any = await this.calculateImpact( - connection, - queryRunner, - sourcingRecord.geoRegionId, - sourcingRecord.materialId, - sourcingRecord.year, - ); - - return { - sourcingRecordId: sourcingRecord.sourcingRecordId, - tonnage: sourcingRecord.tonnage, - year: sourcingRecord.year, - - sourcingLocationId: sourcingRecord.sourcingLocationId, - production: rawValues.production, - harvestedArea: rawValues.harvestedArea, + } - rawDeforestation: rawValues.rawDeforestation, - rawBiodiversity: rawValues.rawBiodiversity, - rawCarbon: rawValues.rawCarbon, - rawWater: rawValues.rawWater, + //Indicator FROM and SELECT statements + for (const indicatorType of Object.values(INDICATOR_TYPES)) { + values.addSelect( + this.indicatorSelectStatementGenerator[indicatorType]( + materialH3s, + indicatorH3s, + ), + generateValueAlias(indicatorType), + ); + } - materialH3DataId: sourcingRecord.materialH3DataId, - }; - }), + for (const [indicatorType, indicatorH3Data] of indicatorH3s) { + values.innerJoin( + indicatorH3Data.h3tableName, + `${indicatorType}`, + `"${indicatorType}".h3index = "geoRegion".h3index`, ); + } - await queryRunner.commitTransaction(); + try { + const result: any = await queryRunner.query(values.getQuery()); + if (!result.length) + this.logger.warn( + `Could not retrieve any raw values for georegion ${georegionId},year ${year} and material ${materialId}`, + ); + return result[0]; } catch (err) { - // rollback changes before throwing error - await queryRunner.rollbackTransaction(); + this.logger.error(err); throw err; - } finally { - // release query runner which is manually created - await queryRunner.release(); - } - - if (!result.length) { - throw new Error('No raw impact data could be calculated'); } - - return result; } /** @@ -252,7 +326,7 @@ export class ImpactCalculatorService { * @param indicatorTypes * @param year */ - getIndicatorH3sByTypeAndClosestYear( + private getIndicatorH3sByTypeAndClosestYear( connection: Connection, queryRunner: QueryRunner, indicatorTypes: INDICATOR_TYPES[], @@ -278,16 +352,11 @@ export class ImpactCalculatorService { const map: Map = await previousValue; - try { - const result: any = await queryRunner.query(queryBuilder.getQuery()); + const result: any = await queryRunner.query(queryBuilder.getQuery()); - if (result.length) { - map.set(currentIndicatorType, result[0]); - } - } catch (err) { - console.error(err); + if (result.length) { + map.set(currentIndicatorType, result[0]); } - return map; }, Promise.resolve(new Map()), @@ -305,7 +374,7 @@ export class ImpactCalculatorService { * @param materialId * @param year */ - getAllMaterialH3sByClosestYear( + private getAllMaterialH3sByClosestYear( connection: Connection, queryRunner: QueryRunner, materialId: string, @@ -331,14 +400,10 @@ export class ImpactCalculatorService { .limit(1); const map: Map = await previousValue; - try { - const result: any = await queryRunner.query(queryBuilder.getQuery()); - - if (result.length) { - map.set(currentMaterialToH3Type, result[0]); - } - } catch (err) { - console.error(err); + const result: any = await queryRunner.query(queryBuilder.getQuery()); + + if (result.length) { + map.set(currentMaterialToH3Type, result[0]); } return map; @@ -347,3 +412,138 @@ export class ImpactCalculatorService { ); } } + +/** + * Small helper function to generate the alias for the select statements + * @param prefix + */ +function generateValueAlias( + prefix: MATERIAL_TO_H3_TYPE | INDICATOR_TYPES, +): string { + return `${prefix}_value`; +} + +//// Select Statetement Generators +// These functions generate the SQL statetement with its corresponding formula, for each +// material type and indicator to be supported following the strategy pattern +// The string representation of the material/indicator type enum is used as aliases for the corresponding +// H3 tables +// TODO this part can potentially be refactored, once compared to indicator record value strategies +function producerMaterialSelectStatementGenerator( + materialH3s: Map, +): string { + const producerType: MATERIAL_TO_H3_TYPE = MATERIAL_TO_H3_TYPE.PRODUCER; + const producerColumn: string = materialH3s.get(producerType)!.h3columnName; + return `sum( "${producerType}"."${producerColumn}" )`; +} + +function harvestMaterialSelectStatementGenerator( + materialH3s: Map, +): string { + const harvestType: MATERIAL_TO_H3_TYPE = MATERIAL_TO_H3_TYPE.HARVEST; + const harvestColumn: string = materialH3s.get(harvestType)!.h3columnName; + return `sum( "${harvestType}"."${harvestColumn}" )`; +} + +function bioDiversitySelectStatementGenerator( + materialH3s: Map, + indicatorH3s: Map, +): string { + const deforestType: INDICATOR_TYPES = INDICATOR_TYPES.DEFORESTATION; + const bioType: INDICATOR_TYPES = INDICATOR_TYPES.BIODIVERSITY_LOSS; + const harvestType: MATERIAL_TO_H3_TYPE = MATERIAL_TO_H3_TYPE.HARVEST; + //Check dependencies/requirements in provided data + checkMissingMaterialH3Data(bioType, materialH3s, [harvestType]); + checkMissingIndicatorH3Dependencies(bioType, indicatorH3s); + + const harvestColumn: string = materialH3s.get(harvestType)!.h3columnName; + const deforestColumn: string = indicatorH3s.get(deforestType)!.h3columnName; + const bioColumn: string = indicatorH3s.get(bioType)!.h3columnName; + + return ( + `sum("${harvestType}"."${harvestColumn}" * "${deforestType}"."${deforestColumn}" ` + + `* "${bioType}"."${bioColumn}" * (1/0.0001) )` + ); +} + +function carbonSelectStatementGenerator( + materialH3s: Map, + indicatorH3s: Map, +): string { + const deforestType: INDICATOR_TYPES = INDICATOR_TYPES.DEFORESTATION; + const carbonType: INDICATOR_TYPES = INDICATOR_TYPES.CARBON_EMISSIONS; + const harvestType: MATERIAL_TO_H3_TYPE = MATERIAL_TO_H3_TYPE.HARVEST; + + //Check dependencies/requirements in provided data + checkMissingMaterialH3Data(carbonType, materialH3s, [harvestType]); + checkMissingIndicatorH3Dependencies(carbonType, indicatorH3s); + + const harvestColumn: string = materialH3s.get(harvestType)!.h3columnName; + const deforestColumn: string = indicatorH3s.get(deforestType)!.h3columnName; + const carbonColumn: string = indicatorH3s.get(carbonType)!.h3columnName; + return ( + `sum( "${harvestType}"."${harvestColumn}" * "${deforestType}"."${deforestColumn}" ` + + `* "${carbonType}"."${carbonColumn}" )` + ); +} + +function deforestationSelectStatementGenerator( + materialH3s: Map, + indicatorH3s: Map, +): string { + const deforestType: INDICATOR_TYPES = INDICATOR_TYPES.DEFORESTATION; + const harvestType: MATERIAL_TO_H3_TYPE = MATERIAL_TO_H3_TYPE.HARVEST; + + //Check dependencies/requirements in provided data + checkMissingMaterialH3Data(deforestType, materialH3s, [harvestType]); + checkMissingIndicatorH3Dependencies(deforestType, indicatorH3s); + + const harvestColumn: string = materialH3s.get(harvestType)!.h3columnName; + const deforestColumn: string = indicatorH3s.get(deforestType)!.h3columnName; + return `sum( "${harvestType}"."${harvestColumn}" * "${deforestType}"."${deforestColumn}" )`; +} + +function waterSelectStatementGenerator( + materialH3s: Map, + indicatorH3s: Map, +): string { + const waterType: INDICATOR_TYPES = INDICATOR_TYPES.UNSUSTAINABLE_WATER_USE; + + //Check dependencies/requirements in provided data + //Water doesn't need materials for the calculation + checkMissingIndicatorH3Dependencies(waterType, indicatorH3s); + + const waterColumn: string = indicatorH3s.get(waterType)!.h3columnName; + return `sum( "${waterType}"."${waterColumn}" * 0.001 )`; +} + +/** + * Helper functions that check missing H3Data dependencies for the SQL queries + */ +function checkMissingMaterialH3Data( + indicatorType: INDICATOR_TYPES, + materialH3s: Map, + requiredMaterialTypes: MATERIAL_TO_H3_TYPE[], +): void { + for (const requiredMaterialType of requiredMaterialTypes) { + if (!materialH3s.get(requiredMaterialType)) { + throw new NotFoundException( + `H3 Data of Material of type ${requiredMaterialType} missing for ${indicatorType} raw value calculations`, + ); + } + } +} + +function checkMissingIndicatorH3Dependencies( + indicatorType: INDICATOR_TYPES, + indicatorH3s: Map, +): void { + Indicator.getIndicatorCalculationDependencies(indicatorType, true).forEach( + (value: INDICATOR_TYPES) => { + if (!indicatorH3s.get(value)) + throw new NotFoundException( + `H3 Data of required Indicator of type ${value} missing for ${indicatorType} raw value calculations`, + ); + }, + ); +} diff --git a/api/src/modules/sourcing-records/dto/sourcing-records-with-indicator-raw-data.dto.ts b/api/src/modules/sourcing-records/dto/sourcing-records-with-indicator-raw-data.dto.ts index cf84f0e76c..2531cf9d02 100644 --- a/api/src/modules/sourcing-records/dto/sourcing-records-with-indicator-raw-data.dto.ts +++ b/api/src/modules/sourcing-records/dto/sourcing-records-with-indicator-raw-data.dto.ts @@ -1,3 +1,5 @@ +import { INDICATOR_TYPES } from 'modules/indicators/indicator.entity'; + /** * @description: Interface for typing the response of the DB that retrieves existing sourcing info with * total production, harvest, and raw indicator data, used for calculating a indicator-record @@ -12,10 +14,13 @@ export class SourcingRecordsWithIndicatorRawDataDto { production: number; harvestedArea: number; + // TODO remove this hardcoded fields once the "simpleImportCalculations" feature has been tested/approved rawDeforestation: number; rawBiodiversity: number; rawCarbon: number; rawWater: number; + indicatorValues: Map; + materialH3DataId: string; } diff --git a/api/test/integration/indicator-record/indicator-records.service.spec.ts b/api/test/integration/indicator-record/indicator-records.service.spec.ts index 12b7f4ba66..18ea77e3db 100644 --- a/api/test/integration/indicator-record/indicator-records.service.spec.ts +++ b/api/test/integration/indicator-record/indicator-records.service.spec.ts @@ -56,6 +56,7 @@ import { SupplierRepository } from '../../../src/modules/suppliers/supplier.repo import { GeoRegionRepository } from '../../../src/modules/geo-regions/geo-region.repository'; import { MaterialRepository } from '../../../src/modules/materials/material.repository'; import { CachedDataRepository } from '../../../src/modules/cached-data/cached-data.repository'; +import * as config from 'config'; describe('Indicator Records Service', () => { let indicatorRecordRepository: IndicatorRecordRepository; @@ -468,7 +469,10 @@ describe('Indicator Records Service', () => { ); //ACT - await indicatorRecordService.createIndicatorRecordsForAllSourcingRecords(); + // TODO remove feature flag selection, once the solution has been approved + config.get('featureFlags.simpleImportCalculations') + ? await indicatorRecordService.createIndicatorRecordsForAllSourcingRecordsV2() + : await indicatorRecordService.createIndicatorRecordsForAllSourcingRecords(); //ASSERT const allIndicators = await indicatorRecordRepository.find();