diff --git a/admin/datasource.ts b/admin/datasource.ts index 642c051f..fa59605c 100644 --- a/admin/datasource.ts +++ b/admin/datasource.ts @@ -26,6 +26,9 @@ import { MRV } from "@shared/entities/cost-inputs/mrv.entity.js"; import { ProjectSize } from "@shared/entities/cost-inputs/project-size.entity.js"; import { ValidationCost } from "@shared/entities/cost-inputs/validation.entity.js"; import { ImplementationLaborCost } from "@shared/entities/cost-inputs/implementation-labor-cost.entity.js"; +import { BaseSize } from "@shared/entities/base-size.entity.js"; +import { BaseIncrease } from "@shared/entities/base-increase.entity.js"; +import { ModelAssumptions } from "@shared/entities/model-assumptions.entity.js"; // TODO: If we import the COMMON_DATABASE_ENTITIES from shared, we get an error where DataSouce is not set for a given entity export const ADMINJS_ENTITIES = [ @@ -56,6 +59,9 @@ export const ADMINJS_ENTITIES = [ EcosystemExtent, Project, ImplementationLaborCost, + BaseSize, + BaseIncrease, + ModelAssumptions, ]; export const dataSource = new DataSource({ diff --git a/admin/index.ts b/admin/index.ts index a7458beb..5e0e41a0 100644 --- a/admin/index.ts +++ b/admin/index.ts @@ -30,6 +30,9 @@ import { SequestrationRateResource } from "./resources/sequestration-rate/seques import { ProjectsResource } from "./resources/projects/projects.resource.js"; import { ProjectSizeResource } from "./resources/project-size/project-size.resource.js"; import { ImplementationLaborCostResource } from "./resources/implementation-labor-cost/implementation-labor-cost.resource.js"; +import { BaseSizeResource } from "./resources/base-size/base-size.resource.js"; +import { BaseIncreaseResource } from "./resources/base-increase/base-increase.resource.js"; +import { ModelAssumptionResource } from "./resources/model-assumptions/model-assumptions.resource.js"; AdminJS.registerAdapter({ Database: AdminJSTypeorm.Database, @@ -79,6 +82,9 @@ const start = async () => { SequestrationRateResource, ProjectsResource, ImplementationLaborCostResource, + BaseSizeResource, + BaseIncreaseResource, + ModelAssumptionResource, { resource: Country, name: "Country", diff --git a/admin/resources/base-increase/base-increase.resource.ts b/admin/resources/base-increase/base-increase.resource.ts new file mode 100644 index 00000000..da98a2a7 --- /dev/null +++ b/admin/resources/base-increase/base-increase.resource.ts @@ -0,0 +1,16 @@ +import { ResourceWithOptions } from "adminjs"; +import { BaseIncrease } from "@shared/entities/base-increase.entity.js"; + +export const BaseIncreaseResource: ResourceWithOptions = { + resource: BaseIncrease, + options: { + sort: { + sortBy: "ecosystem", + direction: "desc", + }, + navigation: { + name: "Data Management", + icon: "Database", + }, + }, +}; diff --git a/admin/resources/base-size/base-size.resource.ts b/admin/resources/base-size/base-size.resource.ts new file mode 100644 index 00000000..bc231f19 --- /dev/null +++ b/admin/resources/base-size/base-size.resource.ts @@ -0,0 +1,16 @@ +import { ResourceWithOptions } from "adminjs"; +import { BaseSize } from "@shared/entities/base-size.entity.js"; + +export const BaseSizeResource: ResourceWithOptions = { + resource: BaseSize, + options: { + sort: { + sortBy: "ecosystem", + direction: "desc", + }, + navigation: { + name: "Data Management", + icon: "Database", + }, + }, +}; diff --git a/admin/resources/model-assumptions/model-assumptions.resource.ts b/admin/resources/model-assumptions/model-assumptions.resource.ts new file mode 100644 index 00000000..a39d8b09 --- /dev/null +++ b/admin/resources/model-assumptions/model-assumptions.resource.ts @@ -0,0 +1,16 @@ +import { ResourceWithOptions } from "adminjs"; +import { ModelAssumptions } from "@shared/entities/model-assumptions.entity.js"; + +export const ModelAssumptionResource: ResourceWithOptions = { + resource: ModelAssumptions, + options: { + sort: { + sortBy: "name", + direction: "desc", + }, + navigation: { + name: "Data Management", + icon: "Database", + }, + }, +}; diff --git a/api/src/modules/import/dtos/excel-base-data.dto.ts b/api/src/modules/import/dtos/excel-base-data.dto.ts deleted file mode 100644 index 781ce741..00000000 --- a/api/src/modules/import/dtos/excel-base-data.dto.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { ACTIVITY } from '@shared/entities/activity.enum'; -import { COMMUNITY_CASH_FLOW_TYPES } from '@shared/entities/cost-inputs/community-cash-flow.entity'; -import { ECOSYSTEM } from '@shared/entities/ecosystem.enum'; - -export type ExcelMasterTable = { - country: string; - ecosystem: ECOSYSTEM; - activity: ACTIVITY; - country_code: string; - continent: string; - hdi: number; - project_size_ha: number; - feasibility_analysis: number; - conservation_planning_and_admin: number; - data_collection_and_field_costs: number; - community_representation: number; - blue_carbon_project_planning: number; - establishing_carbon_rights: number; - financing_cost: number; - validation: number; - implementation_labor_planting: number; - implementation_labor_hybrid: number; - implementation_labor_hydrology: number; - monitoring: number; - maintenance: number; - maintenance_duration: number; - carbon_standard_fees: number; - community_benefit_sharing_fund: number; - baseline_reassessment: number; - MRV: number; - long_term_project_operating_cost: number; - ecosystem_extent: number; - ecosystem_extent_historic: number; - ecosystem_loss_rate: number; - restorable_land: number; - // TODO: Since this is the value that is selected in the corresponding tab within different tiers, maybe the naming is a bit confusing, talk with Elena - tier_1_emission_factor: string; - emission_factor_AGB: number; - emission_factor_SOC: number; - sequestration_rate: number; - other_community_cash_flow: COMMUNITY_CASH_FLOW_TYPES; -}; diff --git a/api/src/modules/import/dtos/excel-base-increase.dto.ts b/api/src/modules/import/dtos/excel-base-increase.dto.ts new file mode 100644 index 00000000..31f5c446 --- /dev/null +++ b/api/src/modules/import/dtos/excel-base-increase.dto.ts @@ -0,0 +1,15 @@ +export type ExcelBaseIncrease = { + ecosystem: string; + feasibility_analysis: number; + conservation_planning_and_admin: number; + data_collection_and_field_cost: number; + community_representation: number; + blue_carbon_project_planning: number; + establishing_carbon_rights: number; + financing_cost: number; + validation: number; + monitoring: number; + baseline_reassessment: number; + MRV: number; + long_term_project_operating_cost: number; +}; diff --git a/api/src/modules/import/dtos/excel-base-size.dto.ts b/api/src/modules/import/dtos/excel-base-size.dto.ts new file mode 100644 index 00000000..1d4d0009 --- /dev/null +++ b/api/src/modules/import/dtos/excel-base-size.dto.ts @@ -0,0 +1,19 @@ +export type ExcelBaseSize = { + activity: string; + ecosystem: string; + feasibility_analysis: number; + conservation_planning_and_admin: number; + data_collection_and_field_cost: number; + community_representation: number; + blue_carbon_project_planning: number; + establishing_carbon_rights: number; + financing_cost: number; + validation: number; + implementation_labor_planting: number; + implementation_labor_hybrid: number; + implementation_labor_hydrology: number; + monitoring: number; + baseline_reassessment: number; + MRV: number; + long_term_project_operating_cost: number; +}; diff --git a/api/src/modules/import/dtos/excel-model-assumptions.dto.ts b/api/src/modules/import/dtos/excel-model-assumptions.dto.ts new file mode 100644 index 00000000..66a36ba1 --- /dev/null +++ b/api/src/modules/import/dtos/excel-model-assumptions.dto.ts @@ -0,0 +1,5 @@ +export type ExcelModelAssumptions = { + Assumptions: string; + Units: string; + Value: string; +}; diff --git a/api/src/modules/import/import.repostiory.ts b/api/src/modules/import/import.repostiory.ts index 5146e3ea..5376203f 100644 --- a/api/src/modules/import/import.repostiory.ts +++ b/api/src/modules/import/import.repostiory.ts @@ -24,6 +24,9 @@ import { MonitoringCost } from '@shared/entities/cost-inputs/monitoring.entity'; import { MRV } from '@shared/entities/cost-inputs/mrv.entity'; import { ValidationCost } from '@shared/entities/cost-inputs/validation.entity'; import { ImplementationLaborCost } from '@shared/entities/cost-inputs/implementation-labor-cost.entity'; +import { BaseIncrease } from '@shared/entities/base-increase.entity'; +import { BaseSize } from '@shared/entities/base-size.entity'; +import { ModelAssumptions } from '@shared/entities/model-assumptions.entity'; @Injectable() export class ImportRepository { @@ -54,6 +57,9 @@ export class ImportRepository { sequestrationRate: SequestrationRate[]; emissionFactors: EmissionFactors[]; implementationLaborCost: ImplementationLaborCost[]; + baseSize: BaseSize[]; + baseIncrease: BaseIncrease[]; + modelAssumptions: ModelAssumptions[]; }) { return this.dataSource.transaction(async (manager) => { await manager.save(importData.projects); @@ -84,6 +90,11 @@ export class ImportRepository { await manager.save(importData.restorableLand); await manager.save(importData.sequestrationRate); await manager.save(importData.emissionFactors); + + // Other tables ingestion + await manager.save(importData.baseSize); + await manager.save(importData.baseIncrease); + await manager.save(importData.modelAssumptions); }); } } diff --git a/api/src/modules/import/services/entity.preprocessor.ts b/api/src/modules/import/services/entity.preprocessor.ts index 774b3048..a73a5465 100644 --- a/api/src/modules/import/services/entity.preprocessor.ts +++ b/api/src/modules/import/services/entity.preprocessor.ts @@ -2,7 +2,6 @@ import { ExcelEstablishingCarbonRights } from './../dtos/excel-establishing-carb import { Injectable } from '@nestjs/common'; import { Country } from '@shared/entities/country.entity'; -import { ExcelMasterTable } from '@api/modules/import/dtos/excel-base-data.dto'; import { Project } from '@shared/entities/projects.entity'; import { ExcelProjects } from '@api/modules/import/dtos/excel-projects.dto'; import { ExcelProjectSize } from '@api/modules/import/dtos/excel-project-size.dto'; @@ -64,6 +63,12 @@ import { ProjectSize } from '@shared/entities/cost-inputs/project-size.entity'; import { ValidationCost } from '@shared/entities/cost-inputs/validation.entity'; import { ExcelImplementationLaborCost } from '../dtos/excel-implementation-labor.dto'; import { ImplementationLaborCost } from '@shared/entities/cost-inputs/implementation-labor-cost.entity'; +import { ExcelBaseSize } from '../dtos/excel-base-size.dto'; +import { ExcelBaseIncrease } from '../dtos/excel-base-increase.dto'; +import { ExcelModelAssumptions } from '../dtos/excel-model-assumptions.dto'; +import { BaseSize } from '@shared/entities/base-size.entity'; +import { BaseIncrease } from '@shared/entities/base-increase.entity'; +import { ModelAssumptions } from '@shared/entities/model-assumptions.entity'; export type ParsedDBEntities = { projects: Project[]; @@ -90,12 +95,14 @@ export type ParsedDBEntities = { sequestrationRate: SequestrationRate[]; emissionFactors: EmissionFactors[]; implementationLaborCost: ImplementationLaborCost[]; + baseSize: BaseSize[]; + baseIncrease: BaseIncrease[]; + modelAssumptions: ModelAssumptions[]; }; @Injectable() export class EntityPreprocessor { toDbEntities(raw: { - master_table: ExcelMasterTable[]; Projects: ExcelProjects[]; 'Project size': ExcelProjectSize[]; 'Feasibility analysis': ExcelFeasibilityAnalysis[]; @@ -120,6 +127,9 @@ export class EntityPreprocessor { 'Sequestration rate': ExcelSequestrationRate[]; 'Emission factors': ExcelEmissionFactors[]; 'Implementation labor': ExcelImplementationLaborCost[]; + base_size_table: ExcelBaseSize[]; + base_increase: ExcelBaseIncrease[]; + 'Model assumptions': ExcelModelAssumptions[]; }): ParsedDBEntities { const processedProjects = this.processProjects(raw.Projects); @@ -181,6 +191,13 @@ export class EntityPreprocessor { raw['Emission factors'], ); + // process other data + const baseSize = this.processBaseSize(raw.base_size_table); + const baseIncrease = this.processBaseIncrease(raw.base_increase); + const modelAssumptions = this.processModelAssumptions( + raw['Model assumptions'], + ); + return { projects: processedProjects, projectSize: projectSize, @@ -206,9 +223,110 @@ export class EntityPreprocessor { sequestrationRate: sequestrationRate, emissionFactors: emissionFactors, implementationLaborCost: implementationLaborCost, + baseSize: baseSize, + baseIncrease: baseIncrease, + modelAssumptions: modelAssumptions, }; } + private processModelAssumptions(raw: ExcelModelAssumptions[]) { + const parsedArray: ModelAssumptions[] = []; + raw.forEach((row: ExcelModelAssumptions) => { + const modelAssumption = new ModelAssumptions(); + modelAssumption.name = row['Assumptions']; + modelAssumption.unit = row['Units']; + modelAssumption.value = row['Value']; + parsedArray.push(modelAssumption); + }); + return parsedArray; + } + + private processBaseIncrease(raw: ExcelBaseIncrease[]) { + const parsedArray: BaseIncrease[] = []; + raw.forEach((row: ExcelBaseIncrease) => { + const baseIncrease = new BaseIncrease(); + baseIncrease.ecosystem = row.ecosystem as ECOSYSTEM; + baseIncrease.feasibilityAnalysis = this.stringToNumeric( + row['feasibility_analysis'], + ); + baseIncrease.conservationPlanningAndAdmin = this.stringToNumeric( + row['conservation_planning_and_admin'], + ); + baseIncrease.dataCollectionAndFieldCost = this.stringToNumeric( + row['data_collection_and_field_cost'], + ); + baseIncrease.communityRepresentation = this.stringToNumeric( + row['community_representation'], + ); + baseIncrease.blueCarbonProjectPlanning = this.stringToNumeric( + row['blue_carbon_project_planning'], + ); + baseIncrease.establishingCarbonRights = this.stringToNumeric( + row['establishing_carbon_rights'], + ); + baseIncrease.financingCost = this.stringToNumeric(row['financing_cost']); + baseIncrease.validation = this.stringToNumeric(row['validation']); + baseIncrease.monitoring = this.stringToNumeric(row['monitoring']); + baseIncrease.baselineReassessment = this.stringToNumeric( + row['baseline_reassessment'], + ); + baseIncrease.mrv = this.stringToNumeric(row['MRV']); + baseIncrease.longTermProjectOperatingCost = this.stringToNumeric( + row['long_term_project_operating_cost'], + ); + parsedArray.push(baseIncrease); + }); + return parsedArray; + } + + private processBaseSize(raw: ExcelBaseSize[]) { + const parsedArray: BaseSize[] = []; + raw.forEach((row: ExcelBaseSize) => { + const baseSize = new BaseSize(); + baseSize.ecosystem = row.ecosystem as ECOSYSTEM; + baseSize.activity = row.activity as ACTIVITY; + baseSize.feasibilityAnalysis = this.stringToNumeric( + row.feasibility_analysis, + ); + baseSize.conservationPlanningAndAdmin = this.stringToNumeric( + row['conservation_planning_and_admin'], + ); + baseSize.dataCollectionAndFieldCost = this.stringToNumeric( + row['data_collection_and_field_cost'], + ); + baseSize.communityRepresentation = this.stringToNumeric( + row['community_representation'], + ); + baseSize.blueCarbonProjectPlanning = this.stringToNumeric( + row['blue_carbon_project_planning'], + ); + baseSize.establishingCarbonRights = this.stringToNumeric( + row['establishing_carbon_rights'], + ); + baseSize.financingCost = this.stringToNumeric(row['financing_cost']); + baseSize.validation = this.stringToNumeric(row['validation']); + baseSize.implementationLaborPlanting = this.stringToNumeric( + row['implementation_labor_planting'], + ); + baseSize.implementationLaborHybrid = this.stringToNumeric( + row['implementation_labor_hybrid'], + ); + baseSize.implementationLaborHydrology = this.stringToNumeric( + row['implementation_labor_hydrology'], + ); + baseSize.monitoring = this.stringToNumeric(row['monitoring']); + baseSize.baselineReassessment = this.stringToNumeric( + row['baseline_reassessment'], + ); + baseSize.mrv = this.stringToNumeric(row['MRV']); + baseSize.longTermProjectOperatingCost = this.stringToNumeric( + row['long_term_project_operating_cost'], + ); + parsedArray.push(baseSize); + }); + return parsedArray; + } + private processImplementationLaborCost(raw: ExcelImplementationLaborCost[]) { const parsedArray: ImplementationLaborCost[] = []; raw.forEach((row: ExcelImplementationLaborCost) => { diff --git a/api/src/modules/import/services/excel-parser.interface.ts b/api/src/modules/import/services/excel-parser.interface.ts index 642be941..0f9bb1d6 100644 --- a/api/src/modules/import/services/excel-parser.interface.ts +++ b/api/src/modules/import/services/excel-parser.interface.ts @@ -25,6 +25,9 @@ export const SHEETS_TO_PARSE = [ 'Sequestration rate', 'Emission factors', 'Implementation labor', + 'base_size_table', + 'base_increase', + 'Model assumptions', ] as const; export interface ExcelParserInterface { diff --git a/api/src/modules/import/services/xlsx.parser.ts b/api/src/modules/import/services/xlsx.parser.ts index 3cb2ff27..d69ba019 100644 --- a/api/src/modules/import/services/xlsx.parser.ts +++ b/api/src/modules/import/services/xlsx.parser.ts @@ -4,13 +4,10 @@ import { ExcelParserInterface, SHEETS_TO_PARSE, } from './excel-parser.interface'; -import { ExcelMasterTable } from '@api/modules/import/dtos/excel-base-data.dto'; @Injectable() export class XlsxParser implements ExcelParserInterface { - async parseExcel( - buffer: Buffer, - ): Promise<{ master_table: ExcelMasterTable[]; Projects: any[] }> { + async parseExcel(buffer: Buffer) { const workbook: WorkBook = read(buffer); const parsedData: any = {}; diff --git a/data/excel/data_ingestion_WIP.xlsm b/data/excel/data_ingestion_WIP.xlsm index e8772f9f..9f68a991 100644 Binary files a/data/excel/data_ingestion_WIP.xlsm and b/data/excel/data_ingestion_WIP.xlsm differ diff --git a/shared/entities/base-increase.entity.ts b/shared/entities/base-increase.entity.ts new file mode 100644 index 00000000..553ad91f --- /dev/null +++ b/shared/entities/base-increase.entity.ts @@ -0,0 +1,47 @@ +import { Entity, Column, PrimaryGeneratedColumn, BaseEntity } from "typeorm"; +import { ECOSYSTEM } from "@shared/entities/ecosystem.enum"; + +@Entity("base_increase") +export class BaseIncrease extends BaseEntity { + @PrimaryGeneratedColumn("uuid") + id: string; + + @Column({ name: "ecosystem", enum: ECOSYSTEM, type: "enum" }) + ecosystem: ECOSYSTEM; + + @Column("decimal", { name: "feasibility_analysis" }) + feasibilityAnalysis: number; + + @Column("decimal", { name: "conservation_planning_and_admin" }) + conservationPlanningAndAdmin: number; + + @Column("decimal", { name: "data_collection_and_field_cost" }) + dataCollectionAndFieldCost: number; + + @Column("decimal", { name: "community_representation" }) + communityRepresentation: number; + + @Column("decimal", { name: "blue_carbon_project_planning" }) + blueCarbonProjectPlanning: number; + + @Column("decimal", { name: "establishing_carbon_rights" }) + establishingCarbonRights: number; + + @Column("decimal", { name: "financing_cost" }) + financingCost: number; + + @Column("decimal", { name: "validation" }) + validation: number; + + @Column("decimal", { name: "monitoring" }) + monitoring: number; + + @Column("decimal", { name: "baseline_reassessment" }) + baselineReassessment: number; + + @Column("decimal", { name: "mrv" }) + mrv: number; + + @Column("decimal", { name: "long_term_project_operating_cost" }) + longTermProjectOperatingCost: number; +} diff --git a/shared/entities/base-size.entity.ts b/shared/entities/base-size.entity.ts new file mode 100644 index 00000000..bb260042 --- /dev/null +++ b/shared/entities/base-size.entity.ts @@ -0,0 +1,60 @@ +import { Entity, Column, PrimaryGeneratedColumn, BaseEntity } from "typeorm"; +import { ECOSYSTEM } from "@shared/entities/ecosystem.enum"; +import { ACTIVITY } from "@shared/entities/activity.enum"; + +@Entity("base_size") +export class BaseSize extends BaseEntity { + @PrimaryGeneratedColumn("uuid") + id: string; + + @Column({ name: "ecosystem", enum: ECOSYSTEM, type: "enum" }) + ecosystem: ECOSYSTEM; + + @Column({ name: "activity", enum: ACTIVITY, type: "enum" }) + activity: ACTIVITY; + + @Column("decimal", { name: "feasibility_analysis" }) + feasibilityAnalysis: number; + + @Column("decimal", { name: "conservation_planning_and_admin" }) + conservationPlanningAndAdmin: number; + + @Column("decimal", { name: "data_collection_and_field_cost" }) + dataCollectionAndFieldCost: number; + + @Column("decimal", { name: "community_representation" }) + communityRepresentation: number; + + @Column("decimal", { name: "blue_carbon_project_planning" }) + blueCarbonProjectPlanning: number; + + @Column("decimal", { name: "establishing_carbon_rights" }) + establishingCarbonRights: number; + + @Column("decimal", { name: "financing_cost" }) + financingCost: number; + + @Column("decimal", { name: "validation" }) + validation: number; + + @Column("decimal", { name: "implementation_labor_planting" }) + implementationLaborPlanting: number; + + @Column("decimal", { name: "implementation_labor_hybrid" }) + implementationLaborHybrid: number; + + @Column("decimal", { name: "implementation_labor_hydrology" }) + implementationLaborHydrology: number; + + @Column("decimal", { name: "monitoring" }) + monitoring: number; + + @Column("decimal", { name: "baseline_reassessment" }) + baselineReassessment: number; + + @Column("decimal", { name: "mrv" }) + mrv: number; + + @Column("decimal", { name: "long_term_project_operating_cost" }) + longTermProjectOperatingCost: number; +} diff --git a/shared/entities/model-assumptions.entity.ts b/shared/entities/model-assumptions.entity.ts new file mode 100644 index 00000000..a8299ede --- /dev/null +++ b/shared/entities/model-assumptions.entity.ts @@ -0,0 +1,16 @@ +import { Entity, Column, PrimaryGeneratedColumn, BaseEntity } from "typeorm"; + +@Entity("model_assumptions") +export class ModelAssumptions extends BaseEntity { + @PrimaryGeneratedColumn("uuid") + id: string; + + @Column("varchar", { name: "name" }) + name: string; + + @Column("varchar", { name: "unit", nullable: true }) + unit: string; + + @Column("varchar", { name: "value" }) + value: string; +} diff --git a/shared/lib/db-entities.ts b/shared/lib/db-entities.ts index ede8868f..89b667ca 100644 --- a/shared/lib/db-entities.ts +++ b/shared/lib/db-entities.ts @@ -26,6 +26,9 @@ import { Country } from "@shared/entities/country.entity"; import { ImplementationLaborCost } from "@shared/entities/cost-inputs/implementation-labor-cost.entity"; import { Project } from "@shared/entities/projects.entity"; import { User } from "@shared/entities/users/user.entity"; +import { BaseSize } from "@shared/entities/base-size.entity"; +import { BaseIncrease } from "@shared/entities/base-increase.entity"; +import { ModelAssumptions } from "@shared/entities/model-assumptions.entity"; export const COMMON_DATABASE_ENTITIES = [ User, @@ -56,4 +59,7 @@ export const COMMON_DATABASE_ENTITIES = [ EcosystemExtent, BaseDataView, ImplementationLaborCost, + BaseSize, + BaseIncrease, + ModelAssumptions, ];