From 72500d2d0611a47b08867676bd734748a57da58b Mon Sep 17 00:00:00 2001 From: alexeh Date: Sun, 29 Sep 2024 11:07:08 +0200 Subject: [PATCH] WIP - inserting data --- api/src/modules/data/data.module.ts | 1 + api/src/modules/data/ecosystem-data.entity.ts | 126 +++++++++++++++--- .../modules/data/ecosystem-data.repository.ts | 24 ++-- api/src/modules/import/import.controller.ts | 10 +- api/src/modules/import/import.module.ts | 4 +- api/src/modules/import/import.service.ts | 24 +++- .../modules/import/services/xlsx.parser.ts | 27 +++- 7 files changed, 171 insertions(+), 45 deletions(-) diff --git a/api/src/modules/data/data.module.ts b/api/src/modules/data/data.module.ts index 6b9de266..6a28aac8 100644 --- a/api/src/modules/data/data.module.ts +++ b/api/src/modules/data/data.module.ts @@ -6,5 +6,6 @@ import { EcoSystemDataRepository } from '@api/modules/data/ecosystem-data.reposi @Module({ imports: [TypeOrmModule.forFeature([EcosystemProject])], providers: [EcoSystemDataRepository], + exports: [EcoSystemDataRepository], }) export class DataModule {} diff --git a/api/src/modules/data/ecosystem-data.entity.ts b/api/src/modules/data/ecosystem-data.entity.ts index ffece133..6535f369 100644 --- a/api/src/modules/data/ecosystem-data.entity.ts +++ b/api/src/modules/data/ecosystem-data.entity.ts @@ -14,26 +14,32 @@ export class EcosystemProject { @Column({ name: 'activity', length: 50 }) activity: string; - @Column({ name: 'country_code', length: 3 }) + @Column({ name: 'country_code', length: 3, nullable: true }) countryCode: string; @Column({ name: 'continent', length: 20 }) continent: string; - @Column('decimal', { name: 'hdi', precision: 3, scale: 2 }) + @Column('decimal', { name: 'hdi', precision: 3, scale: 2, nullable: true }) hdi: number; - @Column('int', { name: 'project_size_ha' }) + @Column('int', { name: 'project_size_ha', nullable: true }) projectSizeHa: number; // TODO: There is a typo in the excel, update both - @Column('decimal', { name: 'feseability_analysis', precision: 10, scale: 2 }) + @Column('decimal', { + name: 'feseability_analysis', + precision: 10, + scale: 2, + nullable: true, + }) feseabilityAnalysis: number; @Column('decimal', { name: 'conservation_planning_and_admin', precision: 10, scale: 2, + nullable: true, }) conservationPlanningAndAdmin: number; @@ -41,6 +47,7 @@ export class EcosystemProject { name: 'data_collection_and_field_costs', precision: 10, scale: 2, + nullable: true, }) dataCollectionAndFieldCosts: number; @@ -48,29 +55,47 @@ export class EcosystemProject { name: 'community_representation', precision: 10, scale: 2, + nullable: true, }) communityRepresentation: number; - @Column('decimal', { name: 'blue_carbon_planning', precision: 10, scale: 2 }) + @Column('decimal', { + name: 'blue_carbon_planning', + precision: 10, + scale: 2, + nullable: true, + }) blueCarbonPlanning: number; @Column('decimal', { name: 'establishing_carbon_rights', precision: 10, scale: 2, + nullable: true, }) establishingCarbonRights: number; - @Column('decimal', { name: 'financing_cost', precision: 5, scale: 4 }) + @Column('decimal', { + name: 'financing_cost', + precision: 5, + scale: 4, + nullable: true, + }) financingCost: number; - @Column('decimal', { name: 'validation', precision: 10, scale: 2 }) + @Column('decimal', { + name: 'validation', + precision: 10, + scale: 2, + nullable: true, + }) validation: number; @Column('decimal', { name: 'implementation_labor_planting', precision: 10, scale: 2, + nullable: true, }) implementationLaborPlanting: number; @@ -78,6 +103,7 @@ export class EcosystemProject { name: 'implementation_labor_hybrid', precision: 10, scale: 2, + nullable: true, }) implementationLaborHybrid: number; @@ -85,69 +111,127 @@ export class EcosystemProject { name: 'implementation_labor_hydrology', precision: 10, scale: 2, + nullable: true, }) implementationLaborHydrology: number; - @Column('decimal', { name: 'monitoring', precision: 10, scale: 2 }) + @Column('decimal', { + name: 'monitoring', + precision: 10, + scale: 2, + nullable: true, + }) monitoring: number; - @Column('decimal', { name: 'maintenance', precision: 5, scale: 4 }) + @Column('decimal', { + name: 'maintenance', + precision: 5, + scale: 4, + nullable: true, + }) maintenance: number; - @Column('smallint', { name: 'maintenance_duration' }) + @Column('smallint', { name: 'maintenance_duration', nullable: true }) maintenanceDuration: number; - @Column('decimal', { name: 'carbon_standard_fees', precision: 5, scale: 4 }) + @Column('decimal', { + name: 'carbon_standard_fees', + precision: 5, + scale: 4, + nullable: true, + }) carbonStandardFees: number; @Column('decimal', { name: 'community_benefit_sharing_fund', precision: 5, scale: 4, + nullable: true, }) communityBenefitSharingFund: number; - @Column('decimal', { name: 'baseline_reassessment', precision: 10, scale: 2 }) + @Column('decimal', { + name: 'baseline_reassessment', + precision: 10, + scale: 2, + nullable: true, + }) baselineReassessment: number; - @Column('decimal', { name: 'MRV', precision: 10, scale: 2 }) + @Column('decimal', { name: 'MRV', precision: 10, scale: 2, nullable: true }) mrv: number; @Column('decimal', { name: 'long_term_project_operating_cost', precision: 10, scale: 2, + nullable: true, }) longTermProjectOperatingCost: number; - @Column('decimal', { name: 'ecosystem_extent', precision: 12, scale: 4 }) + @Column('decimal', { + name: 'ecosystem_extent', + precision: 12, + scale: 4, + nullable: true, + }) ecosystemExtent: number; @Column('decimal', { name: 'ecosystem_extent_historic', precision: 12, scale: 4, + nullable: true, }) ecosystemExtentHistoric: number; - @Column('decimal', { name: 'ecosystem_loss_rate', precision: 10, scale: 9 }) + @Column('decimal', { + name: 'ecosystem_loss_rate', + precision: 10, + scale: 9, + nullable: true, + }) ecosystemLossRate: number; - @Column('decimal', { name: 'restorable_land', precision: 10, scale: 4 }) + @Column('decimal', { + name: 'restorable_land', + precision: 10, + scale: 4, + nullable: true, + }) restorableLand: number; - @Column({ name: 'tier_1_emission_factor', length: 50, nullable: true }) + @Column({ + name: 'tier_1_emission_factor', + length: 50, + nullable: true, + }) tier1EmissionFactor: string; - @Column('decimal', { name: 'emission_factor_AGB', precision: 10, scale: 8 }) + @Column('decimal', { + name: 'emission_factor_AGB', + precision: 10, + scale: 8, + nullable: true, + }) emissionFactorAgb: number; - @Column('decimal', { name: 'emission_factor_SOC', precision: 10, scale: 8 }) + @Column('decimal', { + name: 'emission_factor_SOC', + precision: 10, + scale: 8, + nullable: true, + }) emissionFactorSoc: number; - @Column('decimal', { name: 'sequestration_rate', precision: 8, scale: 4 }) + @Column('decimal', { + name: 'sequestration_rate', + precision: 8, + scale: 4, + nullable: true, + }) sequestrationRate: number; - @Column({ name: 'other_community_cash_flow', length: 50 }) + @Column({ name: 'other_community_cash_flow', length: 50, nullable: true }) otherCommunityCashFlow: string; } diff --git a/api/src/modules/data/ecosystem-data.repository.ts b/api/src/modules/data/ecosystem-data.repository.ts index 58543d3f..02fce024 100644 --- a/api/src/modules/data/ecosystem-data.repository.ts +++ b/api/src/modules/data/ecosystem-data.repository.ts @@ -6,24 +6,28 @@ import { Injectable } from '@nestjs/common'; export class EcoSystemDataRepository extends Repository { columns = this.metadata.columns .filter((c) => !c.isPrimary) - .map((c) => c.propertyName); + .map((c) => c.databaseName); constructor(private datasource: DataSource) { super(EcosystemProject, datasource.createEntityManager()); + console.log('COLUMNSS', this.columns); } /** * @description Insert data into the database and performs an upsert if the data already exists - * @todo: We are now using all columns to determine if the data already exists, define which columns are relevant here, as ID is generated by the database and cant be used */ async insertData(data: EcosystemProject[]): Promise { - return this.createQueryBuilder() - .insert() - .into(EcosystemProject) - .values(data) - .orUpdate(this.columns, this.columns, { - skipUpdateIfNoValuesChanged: true, - }) - .execute(); + return ( + this.createQueryBuilder() + .insert() + .into(EcosystemProject) + .values(data) + // TODO: define what combination of columns might determine if its an update or a new record + // an option would be to use the index of the excel, but this might be a problematic approach + // also, take in consideration concurrency: update this table while other resources are reading from it + // maybe, using a transaction is enough and there is no need for optimistic persimitic locking + //.orUpdate(this.columns) + .execute() + ); } } diff --git a/api/src/modules/import/import.controller.ts b/api/src/modules/import/import.controller.ts index 5f72f19c..47e1e126 100644 --- a/api/src/modules/import/import.controller.ts +++ b/api/src/modules/import/import.controller.ts @@ -10,16 +10,12 @@ import { RolesGuard } from '@api/modules/auth/guards/roles.guard'; import { RequiredRoles } from '@api/modules/auth/decorators/roles.decorator'; import { ROLES } from '@api/modules/auth/roles.enum'; import { UploadXlsm } from '@api/modules/import/decorators/xlsm-upload.decorator'; -import { - ExcelParserInterface, - ExcelParserToken, -} from '@api/modules/import/services/excel-parser.interface'; -import { XlsxParser } from '@api/modules/import/services/xlsx.parser'; +import { ImportService } from '@api/modules/import/import.service'; @Controller() //@UseInterceptors(JwtAuthGuard, RolesGuard) export class ImportController { - constructor(private readonly parser: XlsxParser) {} + constructor(private readonly service: ImportService) {} // TODO: File validation following: // https://docs.nestjs.com/techniques/file-upload @@ -27,6 +23,6 @@ export class ImportController { //@RequiredRoles(ROLES.ADMIN) @UseInterceptors(FileInterceptor('file')) async uploadFile(@UploadXlsm() file: Express.Multer.File): Promise { - return this.parser.parseExcel(file.buffer); + return this.service.import(file); } } diff --git a/api/src/modules/import/import.module.ts b/api/src/modules/import/import.module.ts index b38b1d15..4c00d897 100644 --- a/api/src/modules/import/import.module.ts +++ b/api/src/modules/import/import.module.ts @@ -2,11 +2,11 @@ import { Module } from '@nestjs/common'; import { ImportService } from './import.service'; import { MulterModule } from '@nestjs/platform-express'; import { ImportController } from '@api/modules/import/import.controller'; - import { XlsxParser } from '@api/modules/import/services/xlsx.parser'; +import { DataModule } from '@api/modules/data/data.module'; @Module({ - imports: [MulterModule.register({})], + imports: [MulterModule.register({}), DataModule], controllers: [ImportController], providers: [ImportService, XlsxParser], }) diff --git a/api/src/modules/import/import.service.ts b/api/src/modules/import/import.service.ts index dc6bac11..f62d017e 100644 --- a/api/src/modules/import/import.service.ts +++ b/api/src/modules/import/import.service.ts @@ -1,4 +1,26 @@ import { Injectable } from '@nestjs/common'; +import { XlsxParser } from '@api/modules/import/services/xlsx.parser'; +import { EcoSystemDataRepository } from '@api/modules/data/ecosystem-data.repository'; +import { EcosystemProject } from '@api/modules/data/ecosystem-data.entity'; @Injectable() -export class ImportService {} +export class ImportService { + constructor( + private readonly xlsxParser: XlsxParser, + private readonly repo: EcoSystemDataRepository, + ) {} + + async import(file: Express.Multer.File) { + let data; + try { + data = data = await this.xlsxParser.parseExcel( + file.buffer, + ); + await this.repo.insertData(data); + } catch (e) { + console.log(e); + } finally { + return data; + } + } +} diff --git a/api/src/modules/import/services/xlsx.parser.ts b/api/src/modules/import/services/xlsx.parser.ts index e2fbfe4a..044491be 100644 --- a/api/src/modules/import/services/xlsx.parser.ts +++ b/api/src/modules/import/services/xlsx.parser.ts @@ -1,13 +1,32 @@ -import { ExcelParserInterface } from '@api/modules/import/services/excel-parser.interface'; import { Injectable } from '@nestjs/common'; import { read, utils, WorkBook, WorkSheet } from 'xlsx'; +import { ExcelParserInterface } from './excel-parser.interface'; @Injectable() export class XlsxParser implements ExcelParserInterface { - async parseExcel(buffer: Buffer): Promise { + async parseExcel(buffer: Buffer): Promise { const workbook: WorkBook = read(buffer); const sheet: WorkSheet = workbook.Sheets['master_table']; - const data: any[] = utils.sheet_to_json(sheet, { raw: true, defval: null }); - return data; + const data: T[] = utils.sheet_to_json(sheet, { + raw: true, + defval: null, + }); + return data.map((row) => this.handleCrap(row)); + } + + // TODO: temporal hack to handle stuff, there are values that are No data that could be null in the excel, and missing values like country code or continent + // double check the entity to update it + + private handleCrap(row: T): T { + return Object.fromEntries( + Object.entries(row).map(([key, value]) => [ + key, + value === 'No data' + ? null + : typeof value === 'string' && !isNaN(Number(value)) + ? Number(value) + : value, + ]), + ) as T; } }