diff --git a/api/src/modules/import/dtos/excel-projects-scorecard.dto .ts b/api/src/modules/import/dtos/excel-projects-scorecard.dto .ts new file mode 100644 index 00000000..396feeb7 --- /dev/null +++ b/api/src/modules/import/dtos/excel-projects-scorecard.dto .ts @@ -0,0 +1,15 @@ +import { ECOSYSTEM } from '@shared/entities/ecosystem.enum'; + +export type ExcelProjectScorecard = { + country: string; + country_code: string; + ecosystem: ECOSYSTEM; + legal_feasibility: number; + implementation_risk_score: number; + availability_of_experienced_labor: number; + security_rating: number; + availability_of_alternative_funding: number; + biodiversity_benefit: number; + social_feasibility: number; + coastal_protection_benefit: number; +}; diff --git a/api/src/modules/import/import.controller.ts b/api/src/modules/import/import.controller.ts index 01f4bdc2..495d15ca 100644 --- a/api/src/modules/import/import.controller.ts +++ b/api/src/modules/import/import.controller.ts @@ -44,6 +44,25 @@ export class ImportController { }); } + @TsRestHandler(adminContract.uploadProjectScorecard) + @UseInterceptors(FileInterceptor('file')) + @RequiredRoles(ROLES.ADMIN) + async uploadProjectScorecard( + @UploadXlsm() file: Express.Multer.File, + @GetUser() user: User, + ): Promise { + return tsRestHandler(adminContract.uploadProjectScorecard, async () => { + const importedData = await this.service.importProjectScorecard( + file.buffer, + user.id, + ); + return { + status: 201, + body: importedData, + }; + }); + } + @UseInterceptors(FilesInterceptor('files', 2)) @RequiredRoles(ROLES.PARTNER, ROLES.ADMIN) @TsRestHandler(usersContract.uploadData) diff --git a/api/src/modules/import/import.repostiory.ts b/api/src/modules/import/import.repostiory.ts index 5376203f..7ca81ed2 100644 --- a/api/src/modules/import/import.repostiory.ts +++ b/api/src/modules/import/import.repostiory.ts @@ -27,11 +27,18 @@ import { ImplementationLaborCost } from '@shared/entities/cost-inputs/implementa import { BaseIncrease } from '@shared/entities/base-increase.entity'; import { BaseSize } from '@shared/entities/base-size.entity'; import { ModelAssumptions } from '@shared/entities/model-assumptions.entity'; +import { ProjectScorecard } from '@shared/entities/project-scorecard.entity'; @Injectable() export class ImportRepository { constructor(private readonly dataSource: DataSource) {} + async importProjectScorecard(projectScorecards: ProjectScorecard[]) { + return this.dataSource.transaction(async (manager) => { + await manager.save(projectScorecards); + }); + } + async ingest(importData: { projects: Project[]; projectSize: ProjectSize[]; diff --git a/api/src/modules/import/import.service.ts b/api/src/modules/import/import.service.ts index b4f5c57f..5fe1cdf2 100644 --- a/api/src/modules/import/import.service.ts +++ b/api/src/modules/import/import.service.ts @@ -36,6 +36,24 @@ export class ImportService { private readonly dataSource: DataSource, ) {} + async importProjectScorecard(fileBuffer: Buffer, userId: string) { + this.logger.warn('Project scorecard file import started...'); + this.registerImportEvent(userId, this.eventMap.STARTED); + try { + const parsedSheets = await this.excelParser.parseExcel(fileBuffer); + const parsedDBEntities = + this.preprocessor.toProjectScorecardDbEntries(parsedSheets); + + await this.importRepo.importProjectScorecard(parsedDBEntities); + + this.logger.warn('Excel file import completed successfully'); + this.registerImportEvent(userId, this.eventMap.SUCCESS); + } catch (e) { + this.logger.error('Excel file import failed', e); + this.registerImportEvent(userId, this.eventMap.FAILED); + } + } + async import(fileBuffer: Buffer, userId: string) { this.logger.warn('Excel file import started...'); this.registerImportEvent(userId, this.eventMap.STARTED); @@ -83,7 +101,7 @@ export class ImportService { await userRestorationInputsRepo.save(mappedRestorationInputs); await userConservationInputsRepo.save(mappedConservationInputs); }); - // + return carbonInputs; } } diff --git a/api/src/modules/import/services/entity.preprocessor.ts b/api/src/modules/import/services/entity.preprocessor.ts index 1f33521d..f654fa02 100644 --- a/api/src/modules/import/services/entity.preprocessor.ts +++ b/api/src/modules/import/services/entity.preprocessor.ts @@ -69,6 +69,9 @@ import { ExcelModelAssumptions } from '../dtos/excel-model-assumptions.dto'; import { BaseSize } from '@shared/entities/base-size.entity'; import { BaseIncrease } from '@shared/entities/base-increase.entity'; import { ModelAssumptions } from '@shared/entities/model-assumptions.entity'; +import { ProjectScorecard } from '@shared/entities/project-scorecard.entity'; +import { ExcelProjectScorecard } from '../dtos/excel-projects-scorecard.dto '; +import { PROJECT_SCORE } from '@shared/entities/project-score.enum'; export type ParsedDBEntities = { projects: Project[]; @@ -102,6 +105,10 @@ export type ParsedDBEntities = { @Injectable() export class EntityPreprocessor { + toProjectScorecardDbEntries(raw: {}): ProjectScorecard[] { + return this.processProjectScorecard(raw['Data_ingestion']); + } + toDbEntities(raw: { Projects: ExcelProjects[]; 'Project size': ExcelProjectSize[]; @@ -1125,6 +1132,60 @@ export class EntityPreprocessor { return parsedArray; } + private processProjectScorecard(raw: ExcelProjectScorecard[]) { + const parsedArray: ProjectScorecard[] = []; + raw.forEach((row: ExcelProjectScorecard) => { + const projectScorecard = new ProjectScorecard(); + projectScorecard.countryCode = row.country_code; + projectScorecard.ecosystem = row.ecosystem; + projectScorecard.financialFeasibility = PROJECT_SCORE.LOW; + projectScorecard.legalFeasibility = this.convertNumberToProjectScore( + row.legal_feasibility, + ); + + projectScorecard.implementationFeasibility = + this.convertNumberToProjectScore(row.implementation_risk_score); + + projectScorecard.socialFeasibility = this.convertNumberToProjectScore( + row.social_feasibility, + ); + + projectScorecard.securityRating = this.convertNumberToProjectScore( + row.security_rating, + ); + + projectScorecard.availabilityOfExperiencedLabor = + this.convertNumberToProjectScore(row.availability_of_experienced_labor); + + projectScorecard.availabilityOfAlternatingFunding = + this.convertNumberToProjectScore( + row.availability_of_alternative_funding, + ); + + projectScorecard.coastalProtectionBenefits = + this.convertNumberToProjectScore(row.coastal_protection_benefit); + projectScorecard.biodiversityBenefit = this.convertNumberToProjectScore( + row.biodiversity_benefit, + ); + + parsedArray.push(projectScorecard); + }); + + return parsedArray; + } + + private convertNumberToProjectScore(value: number): PROJECT_SCORE { + if (value === 1) { + return PROJECT_SCORE.LOW; + } + if (value === 2) { + return PROJECT_SCORE.MEDIUM; + } + if (value === 3) { + return PROJECT_SCORE.HIGH; + } + } + private emptyStringToNull(value: any): any | null { return value || null; } diff --git a/api/src/modules/import/services/excel-parser.interface.ts b/api/src/modules/import/services/excel-parser.interface.ts index 315b8e94..90e1a7f3 100644 --- a/api/src/modules/import/services/excel-parser.interface.ts +++ b/api/src/modules/import/services/excel-parser.interface.ts @@ -28,6 +28,7 @@ export const SHEETS_TO_PARSE = [ 'base_size_table', 'base_increase', 'Model assumptions', + 'Data_ingestion', ] as const; export interface ExcelParserInterface { diff --git a/api/test/integration/import/import-scorecard.spec.ts b/api/test/integration/import/import-scorecard.spec.ts new file mode 100644 index 00000000..bdd53680 --- /dev/null +++ b/api/test/integration/import/import-scorecard.spec.ts @@ -0,0 +1,80 @@ +import { TestManager } from '../../utils/test-manager'; +import { HttpStatus } from '@nestjs/common'; +import { adminContract } from '@shared/contracts/admin.contract'; +import { ROLES } from '@shared/entities/users/roles.enum'; +import * as path from 'path'; +import * as fs from 'fs'; +import { ProjectScorecard } from '@shared/entities/project-scorecard.entity'; + +describe('Import Tests', () => { + let testManager: TestManager; + let testUserToken: string; + const testFilePath = path.join( + __dirname, + '../../../../data/excel/data_ingestion_project_scorecard.xlsm', + ); + const fileBuffer = fs.readFileSync(testFilePath); + + beforeAll(async () => { + testManager = await TestManager.createTestManager(); + }); + + beforeEach(async () => { + const { jwtToken } = await testManager.setUpTestUser(); + testUserToken = jwtToken; + }); + + afterEach(async () => { + await testManager.clearDatabase(); + }); + + afterAll(async () => { + await testManager.close(); + }); + + describe('Import Auth', () => { + it('should throw an error if no file is sent', async () => { + const response = await testManager + .request() + .post(adminContract.uploadProjectScorecard.path) + .set('Authorization', `Bearer ${testUserToken}`) + .send(); + + expect(response.status).toBe(HttpStatus.BAD_REQUEST); + expect(response.body.errors[0].title).toBe('File is required'); + }); + + it('should throw an error if the user is not an admin', async () => { + const nonAdminUser = await testManager + .mocks() + .createUser({ role: ROLES.PARTNER, email: 'testtt@user.com' }); + + const { jwtToken } = await testManager.logUserIn(nonAdminUser); + + const response = await testManager + .request() + .post(adminContract.uploadProjectScorecard.path) + .set('Authorization', `Bearer ${jwtToken}`) + .attach('file', fileBuffer, 'data_ingestion_WIP.xlsm'); + + expect(response.status).toBe(HttpStatus.FORBIDDEN); + }); + }); + describe('Import Data', () => { + it('should import project scorecard data from an excel file', async () => { + await testManager.ingestCountries(); + await testManager + .request() + .post(adminContract.uploadProjectScorecard.path) + .set('Authorization', `Bearer ${testUserToken}`) + .attach('file', fileBuffer, 'data_ingestion_project_scorecard.xlsm'); + + const projectScorecard = await testManager + .getDataSource() + .getRepository(ProjectScorecard) + .find(); + + expect(projectScorecard).toHaveLength(208); + }, 30000); + }); +}); diff --git a/data/excel/data_ingestion_project_scorecard.xlsm b/data/excel/data_ingestion_project_scorecard.xlsm new file mode 100644 index 00000000..0d9ec1bd Binary files /dev/null and b/data/excel/data_ingestion_project_scorecard.xlsm differ diff --git a/shared/contracts/admin.contract.ts b/shared/contracts/admin.contract.ts index 3e3a1365..3129b936 100644 --- a/shared/contracts/admin.contract.ts +++ b/shared/contracts/admin.contract.ts @@ -21,4 +21,12 @@ export const adminContract = contract.router({ }, body: contract.type(), }, + uploadProjectScorecard: { + method: "POST", + path: "/admin/upload/scorecard", + responses: { + 201: contract.type(), + }, + body: contract.type(), + }, }); diff --git a/shared/entities/project-scorecard.entity.ts b/shared/entities/project-scorecard.entity.ts index 43552aa8..f4ba29ce 100644 --- a/shared/entities/project-scorecard.entity.ts +++ b/shared/entities/project-scorecard.entity.ts @@ -12,7 +12,6 @@ import { ECOSYSTEM } from "@shared/entities/ecosystem.enum"; import { PROJECT_SCORE } from "@shared/entities/project-score.enum"; @Entity("project_scorecard") -@Unique(["country", "ecosystem"]) export class ProjectScorecard extends BaseEntity { @PrimaryGeneratedColumn("uuid") id: string; @@ -25,30 +24,52 @@ export class ProjectScorecard extends BaseEntity { @JoinColumn({ name: "country_code" }) country: Country; - @Column({ name: "ecosystem", enum: ECOSYSTEM, type: "enum" }) + @Column({ name: "ecosystem", enum: ECOSYSTEM, nullable: true, type: "enum" }) ecosystem: ECOSYSTEM; - @Column({ name: "financial_feasibility", enum: PROJECT_SCORE, type: "enum" }) + @Column({ + name: "financial_feasibility", + nullable: true, + enum: PROJECT_SCORE, + type: "enum", + }) financialFeasibility: PROJECT_SCORE; - @Column({ name: "legal_feasibility", enum: PROJECT_SCORE, type: "enum" }) + @Column({ + name: "legal_feasibility", + nullable: true, + enum: PROJECT_SCORE, + type: "enum", + }) legalFeasibility: PROJECT_SCORE; @Column({ name: "implementation_feasibility", + nullable: true, enum: PROJECT_SCORE, type: "enum", }) implementationFeasibility: PROJECT_SCORE; - @Column({ name: "social_feasibility", enum: PROJECT_SCORE, type: "enum" }) + @Column({ + name: "social_feasibility", + nullable: true, + enum: PROJECT_SCORE, + type: "enum", + }) socialFeasibility: PROJECT_SCORE; - @Column({ name: "security_rating", enum: PROJECT_SCORE, type: "enum" }) + @Column({ + name: "security_rating", + nullable: true, + enum: PROJECT_SCORE, + type: "enum", + }) securityRating: PROJECT_SCORE; @Column({ name: "availability_of_experienced_labor", + nullable: true, enum: PROJECT_SCORE, type: "enum", }) @@ -56,6 +77,7 @@ export class ProjectScorecard extends BaseEntity { @Column({ name: "availability_of_alternating_funding", + nullable: true, enum: PROJECT_SCORE, type: "enum", }) @@ -63,11 +85,17 @@ export class ProjectScorecard extends BaseEntity { @Column({ name: "coastal_protection_benefits", + nullable: true, enum: PROJECT_SCORE, type: "enum", }) coastalProtectionBenefits: PROJECT_SCORE; - @Column({ name: "biodiversity_benefit", enum: PROJECT_SCORE, type: "enum" }) + @Column({ + name: "biodiversity_benefit", + nullable: true, + enum: PROJECT_SCORE, + type: "enum", + }) biodiversityBenefit: PROJECT_SCORE; }