WIP - inserting data

api/src/modules/data/data.module.ts

@@ -6,5 +6,6 @@ import { EcoSystemDataRepository } from '@api/modules/data/ecosystem-data.reposi
 @Module({
   imports: [TypeOrmModule.forFeature([EcosystemProject])],
   providers: [EcoSystemDataRepository],
+  exports: [EcoSystemDataRepository],
 })
 export class DataModule {}

api/src/modules/data/ecosystem-data.entity.ts

@@ -14,140 +14,224 @@ export class EcosystemProject {
   @Column({ name: 'activity', length: 50 })
   activity: string;
 
-  @Column({ name: 'country_code', length: 3 })
+  @Column({ name: 'country_code', length: 3, nullable: true })
   countryCode: string;
 
   @Column({ name: 'continent', length: 20 })
   continent: string;
 
-  @Column('decimal', { name: 'hdi', precision: 3, scale: 2 })
+  @Column('decimal', { name: 'hdi', precision: 3, scale: 2, nullable: true })
   hdi: number;
 
-  @Column('int', { name: 'project_size_ha' })
+  @Column('int', { name: 'project_size_ha', nullable: true })
   projectSizeHa: number;
 
   // TODO: There is a typo in the excel, update both
-  @Column('decimal', { name: 'feseability_analysis', precision: 10, scale: 2 })
+  @Column('decimal', {
+    name: 'feseability_analysis',
+    precision: 10,
+    scale: 2,
+    nullable: true,
+  })
   feseabilityAnalysis: number;
 
   @Column('decimal', {
     name: 'conservation_planning_and_admin',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   conservationPlanningAndAdmin: number;
 
   @Column('decimal', {
     name: 'data_collection_and_field_costs',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   dataCollectionAndFieldCosts: number;
 
   @Column('decimal', {
     name: 'community_representation',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   communityRepresentation: number;
 
-  @Column('decimal', { name: 'blue_carbon_planning', precision: 10, scale: 2 })
+  @Column('decimal', {
+    name: 'blue_carbon_planning',
+    precision: 10,
+    scale: 2,
+    nullable: true,
+  })
   blueCarbonPlanning: number;
 
   @Column('decimal', {
     name: 'establishing_carbon_rights',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   establishingCarbonRights: number;
 
-  @Column('decimal', { name: 'financing_cost', precision: 5, scale: 4 })
+  @Column('decimal', {
+    name: 'financing_cost',
+    precision: 5,
+    scale: 4,
+    nullable: true,
+  })
   financingCost: number;
 
-  @Column('decimal', { name: 'validation', precision: 10, scale: 2 })
+  @Column('decimal', {
+    name: 'validation',
+    precision: 10,
+    scale: 2,
+    nullable: true,
+  })
   validation: number;
 
   @Column('decimal', {
     name: 'implementation_labor_planting',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   implementationLaborPlanting: number;
 
   @Column('decimal', {
     name: 'implementation_labor_hybrid',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   implementationLaborHybrid: number;
 
   @Column('decimal', {
     name: 'implementation_labor_hydrology',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   implementationLaborHydrology: number;
 
-  @Column('decimal', { name: 'monitoring', precision: 10, scale: 2 })
+  @Column('decimal', {
+    name: 'monitoring',
+    precision: 10,
+    scale: 2,
+    nullable: true,
+  })
   monitoring: number;
 
-  @Column('decimal', { name: 'maintenance', precision: 5, scale: 4 })
+  @Column('decimal', {
+    name: 'maintenance',
+    precision: 5,
+    scale: 4,
+    nullable: true,
+  })
   maintenance: number;
 
-  @Column('smallint', { name: 'maintenance_duration' })
+  @Column('smallint', { name: 'maintenance_duration', nullable: true })
   maintenanceDuration: number;
 
-  @Column('decimal', { name: 'carbon_standard_fees', precision: 5, scale: 4 })
+  @Column('decimal', {
+    name: 'carbon_standard_fees',
+    precision: 5,
+    scale: 4,
+    nullable: true,
+  })
   carbonStandardFees: number;
 
   @Column('decimal', {
     name: 'community_benefit_sharing_fund',
     precision: 5,
     scale: 4,
+    nullable: true,
   })
   communityBenefitSharingFund: number;
 
-  @Column('decimal', { name: 'baseline_reassessment', precision: 10, scale: 2 })
+  @Column('decimal', {
+    name: 'baseline_reassessment',
+    precision: 10,
+    scale: 2,
+    nullable: true,
+  })
   baselineReassessment: number;
 
-  @Column('decimal', { name: 'MRV', precision: 10, scale: 2 })
+  @Column('decimal', { name: 'MRV', precision: 10, scale: 2, nullable: true })
   mrv: number;
 
   @Column('decimal', {
     name: 'long_term_project_operating_cost',
     precision: 10,
     scale: 2,
+    nullable: true,
   })
   longTermProjectOperatingCost: number;
 
-  @Column('decimal', { name: 'ecosystem_extent', precision: 12, scale: 4 })
+  @Column('decimal', {
+    name: 'ecosystem_extent',
+    precision: 12,
+    scale: 4,
+    nullable: true,
+  })
   ecosystemExtent: number;
 
   @Column('decimal', {
     name: 'ecosystem_extent_historic',
     precision: 12,
     scale: 4,
+    nullable: true,
   })
   ecosystemExtentHistoric: number;
 
-  @Column('decimal', { name: 'ecosystem_loss_rate', precision: 10, scale: 9 })
+  @Column('decimal', {
+    name: 'ecosystem_loss_rate',
+    precision: 10,
+    scale: 9,
+    nullable: true,
+  })
   ecosystemLossRate: number;
 
-  @Column('decimal', { name: 'restorable_land', precision: 10, scale: 4 })
+  @Column('decimal', {
+    name: 'restorable_land',
+    precision: 10,
+    scale: 4,
+    nullable: true,
+  })
   restorableLand: number;
 
-  @Column({ name: 'tier_1_emission_factor', length: 50, nullable: true })
+  @Column({
+    name: 'tier_1_emission_factor',
+    length: 50,
+    nullable: true,
+  })
   tier1EmissionFactor: string;
 
-  @Column('decimal', { name: 'emission_factor_AGB', precision: 10, scale: 8 })
+  @Column('decimal', {
+    name: 'emission_factor_AGB',
+    precision: 10,
+    scale: 8,
+    nullable: true,
+  })
   emissionFactorAgb: number;
 
-  @Column('decimal', { name: 'emission_factor_SOC', precision: 10, scale: 8 })
+  @Column('decimal', {
+    name: 'emission_factor_SOC',
+    precision: 10,
+    scale: 8,
+    nullable: true,
+  })
   emissionFactorSoc: number;
 
-  @Column('decimal', { name: 'sequestration_rate', precision: 8, scale: 4 })
+  @Column('decimal', {
+    name: 'sequestration_rate',
+    precision: 8,
+    scale: 4,
+    nullable: true,
+  })
   sequestrationRate: number;
 
-  @Column({ name: 'other_community_cash_flow', length: 50 })
+  @Column({ name: 'other_community_cash_flow', length: 50, nullable: true })
   otherCommunityCashFlow: string;
 }

api/src/modules/data/ecosystem-data.repository.ts

@@ -6,24 +6,28 @@ import { Injectable } from '@nestjs/common';
 export class EcoSystemDataRepository extends Repository<EcosystemProject> {
   columns = this.metadata.columns
     .filter((c) => !c.isPrimary)
-    .map((c) => c.propertyName);
+    .map((c) => c.databaseName);
 
   constructor(private datasource: DataSource) {
     super(EcosystemProject, datasource.createEntityManager());
+    console.log('COLUMNSS', this.columns);
   }
 
   /**
    * @description Insert data into the database and performs an upsert if the data already exists
-   * @todo: We are now using all columns to determine if the data already exists, define which columns are relevant here, as ID is generated by the database and cant be used
    */
   async insertData(data: EcosystemProject[]): Promise<any> {
-    return this.createQueryBuilder()
-      .insert()
-      .into(EcosystemProject)
-      .values(data)
-      .orUpdate(this.columns, this.columns, {
-        skipUpdateIfNoValuesChanged: true,
-      })
-      .execute();
+    return (
+      this.createQueryBuilder()
+        .insert()
+        .into(EcosystemProject)
+        .values(data)
+        // TODO: define what combination of columns might determine if its an update or a new record
+        //     an option would be to use the index of the excel, but this might be a problematic approach
+        //    also, take in consideration concurrency: update this table while other resources are reading from it
+        //     maybe, using a transaction is enough and there is no need for optimistic persimitic locking
+        //.orUpdate(this.columns)
+        .execute()
+    );
   }
 }

api/src/modules/import/import.controller.ts

@@ -10,23 +10,19 @@ import { RolesGuard } from '@api/modules/auth/guards/roles.guard';
 import { RequiredRoles } from '@api/modules/auth/decorators/roles.decorator';
 import { ROLES } from '@api/modules/auth/roles.enum';
 import { UploadXlsm } from '@api/modules/import/decorators/xlsm-upload.decorator';
-import {
-  ExcelParserInterface,
-  ExcelParserToken,
-} from '@api/modules/import/services/excel-parser.interface';
-import { XlsxParser } from '@api/modules/import/services/xlsx.parser';
+import { ImportService } from '@api/modules/import/import.service';
 
 @Controller()
 //@UseInterceptors(JwtAuthGuard, RolesGuard)
 export class ImportController {
-  constructor(private readonly parser: XlsxParser) {}
+  constructor(private readonly service: ImportService) {}
   // TODO: File validation following:
   //       https://docs.nestjs.com/techniques/file-upload
 
   @Post('/admin/upload/xlsx')
   //@RequiredRoles(ROLES.ADMIN)
   @UseInterceptors(FileInterceptor('file'))
   async uploadFile(@UploadXlsm() file: Express.Multer.File): Promise<any> {
-    return this.parser.parseExcel(file.buffer);
+    return this.service.import(file);
   }
 }

api/src/modules/import/import.module.ts

@@ -2,11 +2,11 @@ import { Module } from '@nestjs/common';
 import { ImportService } from './import.service';
 import { MulterModule } from '@nestjs/platform-express';
 import { ImportController } from '@api/modules/import/import.controller';
-
 import { XlsxParser } from '@api/modules/import/services/xlsx.parser';
+import { DataModule } from '@api/modules/data/data.module';
 
 @Module({
-  imports: [MulterModule.register({})],
+  imports: [MulterModule.register({}), DataModule],
   controllers: [ImportController],
   providers: [ImportService, XlsxParser],
 })

api/src/modules/import/import.service.ts

@@ -1,4 +1,26 @@
 import { Injectable } from '@nestjs/common';
+import { XlsxParser } from '@api/modules/import/services/xlsx.parser';
+import { EcoSystemDataRepository } from '@api/modules/data/ecosystem-data.repository';
+import { EcosystemProject } from '@api/modules/data/ecosystem-data.entity';
 
 @Injectable()
-export class ImportService {}
+export class ImportService {
+  constructor(
+    private readonly xlsxParser: XlsxParser,
+    private readonly repo: EcoSystemDataRepository,
+  ) {}
+
+  async import(file: Express.Multer.File) {
+    let data;
+    try {
+      data = data = await this.xlsxParser.parseExcel<EcosystemProject>(
+        file.buffer,
+      );
+      await this.repo.insertData(data);
+    } catch (e) {
+      console.log(e);
+    } finally {
+      return data;
+    }
+  }
+}

api/src/modules/import/services/xlsx.parser.ts

@@ -1,13 +1,32 @@
-import { ExcelParserInterface } from '@api/modules/import/services/excel-parser.interface';
 import { Injectable } from '@nestjs/common';
 import { read, utils, WorkBook, WorkSheet } from 'xlsx';
+import { ExcelParserInterface } from './excel-parser.interface';
 
 @Injectable()
 export class XlsxParser implements ExcelParserInterface {
-  async parseExcel(buffer: Buffer): Promise<any> {
+  async parseExcel<T>(buffer: Buffer): Promise<T[]> {
     const workbook: WorkBook = read(buffer);
     const sheet: WorkSheet = workbook.Sheets['master_table'];
-    const data: any[] = utils.sheet_to_json(sheet, { raw: true, defval: null });
-    return data;
+    const data: T[] = utils.sheet_to_json(sheet, {
+      raw: true,
+      defval: null,
+    });
+    return data.map((row) => this.handleCrap<T>(row));
+  }
+
+  // TODO: temporal hack to handle stuff, there are values that are No data that could be null in the excel, and missing values like country code or continent
+  //       double check the entity to update it
+
+  private handleCrap<T>(row: T): T {
+    return Object.fromEntries(
+      Object.entries(row).map(([key, value]) => [
+        key,
+        value === 'No data'
+          ? null
+          : typeof value === 'string' && !isNaN(Number(value))
+            ? Number(value)
+            : value,
+      ]),
+    ) as T;
   }
 }