From 27d867db3571ae732186769edadea7975f4b7310 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sun, 6 Oct 2024 08:31:23 +0200 Subject: [PATCH 1/3] create scaffold cost input entity --- api/src/modules/model/base-data.entity.ts | 34 +++++++------- .../model/entities/cost-input.entity.ts | 46 +++++++++++++++++++ 2 files changed, 63 insertions(+), 17 deletions(-) create mode 100644 api/src/modules/model/entities/cost-input.entity.ts diff --git a/api/src/modules/model/base-data.entity.ts b/api/src/modules/model/base-data.entity.ts index eef2f738..c02e6b66 100644 --- a/api/src/modules/model/base-data.entity.ts +++ b/api/src/modules/model/base-data.entity.ts @@ -41,23 +41,23 @@ export class BaseData { @Column('int', { name: 'project_size_ha', nullable: true }) projectSizeHa: number; - // // TODO: There is a typo in the excel, update both - // @Column('decimal', { - // name: 'feseability_analysis', - // precision: 10, - // scale: 2, - // nullable: true, - // }) - // feseabilityAnalysis: number; - // - // @Column('decimal', { - // name: 'conservation_planning_and_admin', - // precision: 10, - // scale: 2, - // nullable: true, - // }) - // conservationPlanningAndAdmin: number; - // + // TODO: There is a typo in the excel, update both + @Column('decimal', { + name: 'feasibility_analysis', + precision: 10, + scale: 2, + nullable: true, + }) + feasibilityAnalysis: number; + + @Column('decimal', { + name: 'conservation_planning_and_admin', + precision: 10, + scale: 2, + nullable: true, + }) + conservationPlanningAndAdmin: number; + // @Column('decimal', { // name: 'data_collection_and_field_costs', // precision: 10, diff --git a/api/src/modules/model/entities/cost-input.entity.ts b/api/src/modules/model/entities/cost-input.entity.ts new file mode 100644 index 00000000..0e05c759 --- /dev/null +++ b/api/src/modules/model/entities/cost-input.entity.ts @@ -0,0 +1,46 @@ +import { + Column, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; +import { ACTIVITY, ECOSYSTEM } from '@api/modules/model/base-data.entity'; +import { Country } from '@api/modules/model/entities/country.entity'; + +export enum COST_INPUT_TYPE { + PROJECT_SIZE_HA = 'project_size_ha', + FEASIBILITY_ANALYSIS = 'feasibility_analysis', + CONSERVATION_PLANNING_AND_ADMIN = 'conservation_planning_and_admin', +} + +@Index('idx_country_activity_ecosystem', [ + 'countryCode', + 'activity', + 'ecosystem', +]) +@Entity('cost_input') +export class CostInput { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ name: 'ecosystem', enum: ECOSYSTEM, type: 'enum' }) + ecosystem: ECOSYSTEM; + + @Column({ name: 'activity', enum: ACTIVITY, type: 'enum' }) + activity: ACTIVITY; + + @Column({ name: 'country_code', length: 3, nullable: true }) + countryCode: string; + + @ManyToOne(() => Country) + @JoinColumn({ name: 'country_code' }) + country: Country; + + @Column({ type: 'enum', enum: COST_INPUT_TYPE }) + type: COST_INPUT_TYPE; + + @Column('decimal', { precision: 10, scale: 2 }) + value: number; +} From 8c0b6b3c6753b2a829c1194df3cb73fdfac905a2 Mon Sep 17 00:00:00 2001 From: alexeh Date: Sun, 6 Oct 2024 09:58:39 +0200 Subject: [PATCH 2/3] create scaffold carbon input, update 2 tables from base data --- api/src/modules/config/app-config.service.ts | 4 ++ .../import/services/entity.preprocessor.ts | 13 ++++ api/src/modules/model/base-data.entity.ts | 24 +++++++ api/src/modules/model/base-data.repository.ts | 68 ++++++++++++++++++- .../model/entities/carbon-input.entity.ts | 47 +++++++++++++ .../model/entities/cost-input.entity.ts | 1 + 6 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 api/src/modules/model/entities/carbon-input.entity.ts diff --git a/api/src/modules/config/app-config.service.ts b/api/src/modules/config/app-config.service.ts index 6e8ffe8b..7895ab8e 100644 --- a/api/src/modules/config/app-config.service.ts +++ b/api/src/modules/config/app-config.service.ts @@ -6,6 +6,8 @@ import { JwtConfigHandler } from '@api/modules/config/auth-config.handler'; import { ApiEventsEntity } from '@api/modules/api-events/api-events.entity'; import { BaseData } from '@api/modules/model/base-data.entity'; import { Country } from '@api/modules/model/entities/country.entity'; +import { CostInput } from '@api/modules/model/entities/cost-input.entity'; +import { CarbonInputEntity } from '@api/modules/model/entities/carbon-input.entity'; export type JWTConfig = { secret: string; @@ -37,6 +39,8 @@ export class ApiConfigService { ApiEventsEntity, Country, BaseData, + CostInput, + CarbonInputEntity, ], synchronize: true, ssl: this.isProduction() diff --git a/api/src/modules/import/services/entity.preprocessor.ts b/api/src/modules/import/services/entity.preprocessor.ts index fa6892d0..0fbe007c 100644 --- a/api/src/modules/import/services/entity.preprocessor.ts +++ b/api/src/modules/import/services/entity.preprocessor.ts @@ -38,6 +38,7 @@ export class EntityPreprocessor { }); } + // TODO: type raw base data shape (excel) private processBaseData(rawBaseData: any[]): BaseData[] { return rawBaseData.map((rawData) => { const baseData = new BaseData(); @@ -45,6 +46,18 @@ export class EntityPreprocessor { baseData.activity = rawData.activity; baseData.countryCode = rawData.country_code; baseData.projectSizeHa = this.emptyStringToNull(rawData.project_size_ha); + baseData.feasibilityAnalysis = this.emptyStringToNull( + rawData.feseability_analysis, + ); + baseData.conservationPlanningAndAdmin = this.emptyStringToNull( + rawData.conservation_planning_and_admin, + ); + baseData.ecosystemExtent = this.emptyStringToNull( + rawData.ecosystem_extent, + ); + baseData.ecosystemLoss = this.emptyStringToNull(rawData.ecosystem_loss); + baseData.restorableLand = this.emptyStringToNull(rawData.restorable_land); + return baseData; }); } diff --git a/api/src/modules/model/base-data.entity.ts b/api/src/modules/model/base-data.entity.ts index c02e6b66..385fd802 100644 --- a/api/src/modules/model/base-data.entity.ts +++ b/api/src/modules/model/base-data.entity.ts @@ -58,6 +58,30 @@ export class BaseData { }) conservationPlanningAndAdmin: number; + @Column('decimal', { + name: 'ecosystem_extent', + precision: 12, + scale: 4, + nullable: true, + }) + ecosystemExtent: number; + + @Column('decimal', { + name: 'ecosystem_loss', + precision: 10, + scale: 9, + nullable: true, + }) + ecosystemLoss: number; + + @Column('decimal', { + name: 'restorable_land', + precision: 10, + scale: 4, + nullable: true, + }) + restorableLand: number; + // @Column('decimal', { // name: 'data_collection_and_field_costs', // precision: 10, diff --git a/api/src/modules/model/base-data.repository.ts b/api/src/modules/model/base-data.repository.ts index b7b30abf..3126fe72 100644 --- a/api/src/modules/model/base-data.repository.ts +++ b/api/src/modules/model/base-data.repository.ts @@ -1,22 +1,84 @@ -import { DataSource, Repository } from 'typeorm'; +import { DataSource, EntityManager, Repository } from 'typeorm'; import { Injectable } from '@nestjs/common'; import { ParsedDBEntities } from '@api/modules/import/services/entity.preprocessor'; import { BaseData } from '@api/modules/model/base-data.entity'; import { Country } from '@api/modules/model/entities/country.entity'; +import { + COST_INPUT_TYPE, + CostInput, +} from '@api/modules/model/entities/cost-input.entity'; +import { + CARBON_INPUT_TYPE, + CarbonInputEntity, +} from '@api/modules/model/entities/carbon-input.entity'; @Injectable() export class BaseDataRepository extends Repository { + private fieldMappingCostInput = { + projectSizeHa: COST_INPUT_TYPE.PROJECT_SIZE_HA, + feasibilityAnalysis: COST_INPUT_TYPE.FEASIBILITY_ANALYSIS, + conservationPlanningAndAdmin: + COST_INPUT_TYPE.CONSERVATION_PLANNING_AND_ADMIN, + }; + + private fieldMappingCarbonInput = { + ecosystemExtent: CARBON_INPUT_TYPE.ECOSYSTEM_EXTENT, + ecosystemLoss: CARBON_INPUT_TYPE.ECOSYSTEM_LOSS, + restorableLand: CARBON_INPUT_TYPE.RESTORABLE_LAND, + }; + constructor(private datasource: DataSource) { super(BaseData, datasource.createEntityManager()); } + // TODO: Countries are not duplicated but cost and base data are. Check this + async insertData(data: ParsedDBEntities): Promise { return await this.datasource.transaction(async (manager) => { const countryRepo = manager.getRepository(Country); const baseDataRepo = manager.getRepository(BaseData); - await countryRepo.insert(data.countries); - await baseDataRepo.insert(data.baseData); + await countryRepo.save(data.countries); + const baseData = await baseDataRepo.save(data.baseData); + + return this.updateInputs(manager, baseData); }); } + + async updateInputs(manager: EntityManager, data: BaseData[]) { + const costInputs: CostInput[] = []; + const carbonInputs: CarbonInputEntity[] = []; + for (const baseData of data) { + const { countryCode, ecosystem, activity } = baseData; + for (const field in baseData) { + const value = baseData[field]; + const costInputType = this.fieldMappingCostInput[field]; + const carbonInputType = this.fieldMappingCarbonInput[field]; + if (value) { + if (costInputType) { + const costInput = new CostInput(); + costInput.countryCode = countryCode; + costInput.ecosystem = ecosystem; + costInput.activity = activity; + costInput.type = costInputType; + costInput.value = value; + costInputs.push(costInput); + } + if (carbonInputType) { + const carbonInput = new CarbonInputEntity(); + carbonInput.countryCode = countryCode; + carbonInput.ecosystem = ecosystem; + carbonInput.activity = activity; + carbonInput.type = carbonInputType; + carbonInput.value = value; + carbonInputs.push(carbonInput); + } + } + } + } + const costInputRepo = manager.getRepository(CostInput); + const carbonInputRepo = manager.getRepository(CarbonInputEntity); + await costInputRepo.save(costInputs); + await carbonInputRepo.save(carbonInputs); + } } diff --git a/api/src/modules/model/entities/carbon-input.entity.ts b/api/src/modules/model/entities/carbon-input.entity.ts new file mode 100644 index 00000000..eb032858 --- /dev/null +++ b/api/src/modules/model/entities/carbon-input.entity.ts @@ -0,0 +1,47 @@ +import { + Column, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; +import { ACTIVITY, ECOSYSTEM } from '@api/modules/model/base-data.entity'; +import { Country } from '@api/modules/model/entities/country.entity'; + +export enum CARBON_INPUT_TYPE { + ECOSYSTEM_EXTENT = 'ecosystem_extent', + ECOSYSTEM_LOSS = 'ecoystem_loss', + RESTORABLE_LAND = 'restorable_land', +} + +@Index('idx_carbon_input_country_activity_ecosystem', [ + 'countryCode', + 'activity', + 'ecosystem', +]) +@Entity('carbon_input') +export class CarbonInputEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ name: 'ecosystem', enum: ECOSYSTEM, type: 'enum' }) + ecosystem: ECOSYSTEM; + + @Column({ name: 'activity', enum: ACTIVITY, type: 'enum' }) + activity: ACTIVITY; + + @Column({ name: 'country_code', length: 3, nullable: true }) + countryCode: string; + + @ManyToOne(() => Country) + @JoinColumn({ name: 'country_code' }) + country: Country; + + // TODO: Probably this column shoudl not be editable + @Column({ type: 'enum', enum: CARBON_INPUT_TYPE }) + type: CARBON_INPUT_TYPE; + + @Column('decimal', { precision: 10, scale: 2 }) + value: number; +} diff --git a/api/src/modules/model/entities/cost-input.entity.ts b/api/src/modules/model/entities/cost-input.entity.ts index 0e05c759..2d82f264 100644 --- a/api/src/modules/model/entities/cost-input.entity.ts +++ b/api/src/modules/model/entities/cost-input.entity.ts @@ -38,6 +38,7 @@ export class CostInput { @JoinColumn({ name: 'country_code' }) country: Country; + // TODO: Probably this column shoudl not be editable @Column({ type: 'enum', enum: COST_INPUT_TYPE }) type: COST_INPUT_TYPE; From 452395ee415e7da2e029066563602b17e6a63fff Mon Sep 17 00:00:00 2001 From: alexeh Date: Sun, 6 Oct 2024 10:12:29 +0200 Subject: [PATCH 3/3] fix multer typing issues --- api/package.json | 1 - api/tsconfig.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/api/package.json b/api/package.json index ffb7170b..06f07e5f 100644 --- a/api/package.json +++ b/api/package.json @@ -49,7 +49,6 @@ "@types/express": "^4.17.17", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.7", - "@types/multer": "^1.4.12", "@types/node": "catalog:", "@types/nodemailer": "^6.4.15", "@types/passport-jwt": "^4.0.1", diff --git a/api/tsconfig.json b/api/tsconfig.json index cdcfa9b3..465486dd 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../tsconfig.json", "compilerOptions": { - "types": ["node", "jest", "Multer"], + "types": ["node", "jest"], "module": "commonjs", "declaration": true, "removeComments": true,