Merge pull request #269 from EyeSeeTea/development

Release 1.6.4
EyeSeeTea · Jun 21, 2024 · e0b8ca8 · e0b8ca8
2 parents a993dfd + c14891d
commit e0b8ca8
Show file tree

Hide file tree

Showing 47 changed files with 1,993 additions and 915 deletions.
diff --git a/i18n/en.pot b/i18n/en.pot
@@ -5,8 +5,8 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1)\n"
-"POT-Creation-Date: 2024-04-12T05:19:17.327Z\n"
-"PO-Revision-Date: 2024-04-12T05:19:17.327Z\n"
+"POT-Creation-Date: 2024-06-21T20:57:02.640Z\n"
+"PO-Revision-Date: 2024-06-21T20:57:02.640Z\n"
 
 msgid "Template {{id}} not loaded"
 msgstr ""
@@ -100,12 +100,10 @@ msgstr ""
 msgid "Invalid choice was chosen"
 msgstr ""
 
-msgid ""
-"If ATC code in ATC levels 4 A07AA and P01AB, Route of administration must "
-"be oral"
+msgid "The AST results provided are not valid (they are all empty)"
 msgstr ""
 
-msgid "The AST results provided are not valid (they are all empty)"
+msgid "Import Ignored"
 msgstr ""
 
 msgid "Profile"
@@ -565,6 +563,9 @@ msgid ""
 "can still proceed with the submission."
 msgstr ""
 
+msgid "Download Errors as CSV"
+msgstr ""
+
 msgid "Continue"
 msgstr ""
 
@@ -592,7 +593,7 @@ msgstr ""
 msgid "App support ticket"
 msgstr ""
 
-msgid "Upload New Data Files"
+msgid "Cancel upload"
 msgstr ""
 
 msgid "Fix upload data"

diff --git a/i18n/es.po b/i18n/es.po
@@ -1,7 +1,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: i18next-conv\n"
-"POT-Creation-Date: 2024-04-12T05:19:17.327Z\n"
+"POT-Creation-Date: 2024-06-21T20:57:02.640Z\n"
 "PO-Revision-Date: 2018-10-25T09:02:35.143Z\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -100,12 +100,10 @@ msgstr ""
 msgid "Invalid choice was chosen"
 msgstr ""
 
-msgid ""
-"If ATC code in ATC levels 4 A07AA and P01AB, Route of administration must be "
-"oral"
+msgid "The AST results provided are not valid (they are all empty)"
 msgstr ""
 
-msgid "The AST results provided are not valid (they are all empty)"
+msgid "Import Ignored"
 msgstr ""
 
 msgid "Profile"
@@ -567,6 +565,9 @@ msgid ""
 "can still proceed with the submission."
 msgstr ""
 
+msgid "Download Errors as CSV"
+msgstr ""
+
 msgid "Continue"
 msgstr ""
 
@@ -594,7 +595,7 @@ msgstr ""
 msgid "App support ticket"
 msgstr ""
 
-msgid "Upload New Data Files"
+msgid "Cancel upload"
 msgstr ""
 
 msgid "Fix upload data"

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "glass",
     "description": "DHIS2 Glass App",
-    "version": "1.6.3",
+    "version": "1.6.4",
     "license": "GPL-3.0",
     "author": "EyeSeeTea team",
     "homepage": ".",
@@ -16,7 +16,7 @@
         "@dhis2/d2-i18n-generate": "1.2.0",
         "@dhis2/ui": "6.12.0",
         "@eyeseetea/d2-api": "1.13.2-beta.6",
-        "@eyeseetea/d2-logger": "0.1.1",
+        "@eyeseetea/d2-logger": "1.1.0",
         "@eyeseetea/d2-ui-components": "2.6.11",
         "@eyeseetea/xlsx-populate": "4.1.0",
         "@material-ui/core": "4.12.3",
@@ -115,7 +115,9 @@
         "prepare": "husky install",
         "script-example": "npx ts-node src/scripts/index.ts example getExample",
         "start-amc-recalculate": "npx ts-node src/scripts/cliAMC.ts  --debug",
-        "build-amc-recalculate": "ncc build src/scripts/cliAMC.ts -m -o $npm_package_name-amc-recalculate-server && zip -r $npm_package_name-amc-recalculate-server.zip $npm_package_name-amc-recalculate-server && npx rimraf $npm_package_name-amc-recalculate-server/"
+        "build-amc-recalculate": "ncc build src/scripts/cliAMC.ts -m -o $npm_package_name-amc-recalculate-server && zip -r $npm_package_name-amc-recalculate-server.zip $npm_package_name-amc-recalculate-server && npx rimraf $npm_package_name-amc-recalculate-server/",
+        "amr-agg-data-validation": "ts-node src/scripts/amr_agg_data_validation.ts",
+        "amr-agg-data-reset": "ts-node src/scripts/amr_agg_data_reset.ts"
     },
     "manifest.webapp": {
         "name": "glass",

diff --git a/src/CompositionRoot.ts b/src/CompositionRoot.ts
@@ -276,14 +276,16 @@ export function getCompositionRoot(instance: Instance) {
                 instanceRepository,
                 amcProductDataRepository,
                 glassAtcRepository,
-                metadataRepository
+                metadataRepository,
+                glassModuleRepository
             ),
             consumptionDataSubstanceLevel: new CalculateConsumptionDataSubstanceLevelUseCase(
                 glassUploadsRepository,
                 glassDocumentsRepository,
                 amcSubstanceDataRepository,
                 glassAtcRepository,
-                metadataRepository
+                metadataRepository,
+                glassModuleRepository
             ),
         }),
 

diff --git a/src/data/repositories/GlassATCDefaultRepository.ts b/src/data/repositories/GlassATCDefaultRepository.ts
@@ -31,11 +31,11 @@ export class GlassATCDefaultRepository implements GlassATCRepository {
                 if (atcVersionDataStore) {
                     return Future.success(this.buildGlassAtcVersionDataWithCorrectNames(atcVersionDataStore));
                 }
-                logger.error(`ATC version ${atcVersionKey} not found`);
+                logger.error(`[${new Date().toISOString()}] ATC version ${atcVersionKey} not found`);
                 return Future.error(`ATC version ${atcVersionKey} not found`);
             });
         }
-        logger.error(`ATC version ${atcVersionKey} is not valid`);
+        logger.error(`[${new Date().toISOString()}] ATC version ${atcVersionKey} is not valid`);
         return Future.error(`ATC version ${atcVersionKey} is not valid`);
     }
 
@@ -45,7 +45,7 @@ export class GlassATCDefaultRepository implements GlassATCRepository {
             const atcCurrentVersionInfo = atcVersionHistory.find(({ currentVersion }) => currentVersion);
 
             if (!atcCurrentVersionInfo) {
-                logger.error(`Cannot find current version of ATC`);
+                logger.error(`[${new Date().toISOString()}] Cannot find current version of ATC`);
                 return Future.error("Cannot find current version of ATC");
             }
             const atcVersionKey = createAtcVersionKey(atcCurrentVersionInfo.year, atcCurrentVersionInfo.version);
@@ -64,7 +64,7 @@ export class GlassATCDefaultRepository implements GlassATCRepository {
                         [atcVersionKey]: this.getAtcVersion(atcVersionKey),
                     };
                 }
-                logger.error(`ATC version key not valid: ${atcVersionKey}`);
+                logger.error(`[${new Date().toISOString()}] ATC version key not valid: ${atcVersionKey}`);
                 return acc;
             }, {})
         );

diff --git a/src/data/repositories/data-entry/AMCProductDataDefaultRepository.ts b/src/data/repositories/data-entry/AMCProductDataDefaultRepository.ts
@@ -94,16 +94,26 @@ export class AMCProductDataDefaultRepository implements AMCProductDataRepository
         if (!_.isEmpty(d2TrackerEvents)) {
             return importApiTracker(this.api, { events: d2TrackerEvents }, importStrategy);
         } else {
-            logger.error(`Product level data: there are no events to be created`);
+            logger.error(`[${new Date().toISOString()}] Product level data: there are no events to be created`);
             return Future.error("There are no events to be created");
         }
     }
 
     getProductRegisterAndRawProductConsumptionByProductIds(
         orgUnitId: Id,
         productIds: string[],
-        period: string
+        period: string,
+        productIdsChunkSize: number,
+        chunked?: boolean
     ): FutureData<ProductDataTrackedEntity[]> {
+        if (chunked) {
+            return this.getProductRegisterAndRawProductConsumptionByProductIdsChunked(
+                orgUnitId,
+                productIds,
+                period,
+                productIdsChunkSize
+            );
+        }
         return Future.fromPromise(
             this.getProductRegisterAndRawProductConsumptionByProductIdsAsync(orgUnitId, productIds, period)
         ).map(trackedEntities => {
@@ -133,6 +143,41 @@ export class AMCProductDataDefaultRepository implements AMCProductDataRepository
         });
     }
 
+    private getProductRegisterAndRawProductConsumptionByProductIdsChunked(
+        orgUnit: Id,
+        productIds: string[],
+        period: string,
+        productIdsChunkSize: number
+    ): FutureData<ProductDataTrackedEntity[]> {
+        const chunkedProductIds = _(productIds).chunk(productIdsChunkSize).value();
+        const enrollmentEnrolledAfter = `${period}-1-1`;
+        const enrollmentEnrolledBefore = `${period}-12-31`;
+
+        return Future.sequential(
+            chunkedProductIds.flatMap(productIdsChunk => {
+                const productIdsString = productIdsChunk.join(";");
+                const filter = `${AMR_GLASS_AMC_TEA_PRODUCT_ID}:IN:${productIdsString}`;
+
+                // TODO: change pageSize to skipPaging:true when new version of d2-api
+                return apiToFuture(
+                    this.api.tracker.trackedEntities.get({
+                        fields: trackedEntitiesFields,
+                        program: AMC_PRODUCT_REGISTER_PROGRAM_ID,
+                        programStage: AMC_RAW_PRODUCT_CONSUMPTION_STAGE_ID,
+                        orgUnit: orgUnit,
+                        filter: filter,
+                        enrollmentEnrolledAfter: enrollmentEnrolledAfter,
+                        enrollmentEnrolledBefore: enrollmentEnrolledBefore,
+                        pageSize: productIdsChunk.length,
+                    })
+                ).flatMap((trackedEntitiesResponse: TrackedEntitiesGetResponse) => {
+                    const productData = this.mapFromTrackedEntitiesToProductData(trackedEntitiesResponse.instances);
+                    return Future.success(productData);
+                });
+            })
+        ).flatMap(listOfProductData => Future.success(_(listOfProductData).flatten().value()));
+    }
+
     private async getProductRegisterAndRawProductConsumptionByProductIdsAsync(
         orgUnit: Id,
         productIds: string[],

diff --git a/src/data/repositories/data-entry/AMCSubstanceDataDefaultRepository.ts b/src/data/repositories/data-entry/AMCSubstanceDataDefaultRepository.ts
@@ -62,11 +62,18 @@ export class AMCSubstanceDataDefaultRepository implements AMCSubstanceDataReposi
 
     getRawSubstanceConsumptionDataByEventsIds(
         orgUnitId: Id,
-        eventsIds: Id[]
+        substanceIds: Id[],
+        substanceIdsChunkSize: number,
+        chunked?: boolean
     ): FutureData<RawSubstanceConsumptionData[] | undefined> {
         return Future.joinObj({
             rawSubstanceConsumptionProgram: this.getRawSubstanceConsumptionProgram(),
-            substanceConsumptionDataEvents: this.getRawSubstanceConsumptionDataD2EventsByIds(orgUnitId, eventsIds),
+            substanceConsumptionDataEvents: this.getRawSubstanceConsumptionDataD2EventsByIds(
+                orgUnitId,
+                substanceIds,
+                substanceIdsChunkSize,
+                chunked
+            ),
         }).map(({ rawSubstanceConsumptionProgram, substanceConsumptionDataEvents }) => {
             const programStageDataElements = rawSubstanceConsumptionProgram?.programStages.find(
                 ({ id }) => AMC_RAW_SUBSTANCE_CONSUMPTION_DATA_PROGRAM_STAGE_ID === id
@@ -154,7 +161,7 @@ export class AMCSubstanceDataDefaultRepository implements AMCSubstanceDataReposi
                     });
                 });
             } else {
-                logger.error(`Substance level data: there are no events to be created`);
+                logger.error(`[${new Date().toISOString()}] Substance level data: there are no events to be created`);
                 return Future.error("There are no events to be created");
             }
         });
@@ -252,6 +259,7 @@ export class AMCSubstanceDataDefaultRepository implements AMCSubstanceDataReposi
 
                 if (Object.keys(consumptionData).length) {
                     return {
+                        id: substanceConsumptionDataEvent.event,
                         report_date: substanceConsumptionDataEvent.occurredAt,
                         ...consumptionData,
                     };
@@ -331,12 +339,51 @@ export class AMCSubstanceDataDefaultRepository implements AMCSubstanceDataReposi
         ).map(response => response.objects[0] as D2Program | undefined);
     }
 
-    private getRawSubstanceConsumptionDataD2EventsByIds(orgUnitId: Id, eventsIds: Id[]): FutureData<D2TrackerEvent[]> {
-        return Future.fromPromise(this.getRawSubstanceConsumptionDataByEventsIdsAsync(orgUnitId, eventsIds)).map(
+    private getRawSubstanceConsumptionDataD2EventsByIds(
+        orgUnitId: Id,
+        substanceIds: Id[],
+        substanceIdsChunkSize: number,
+        chunked?: boolean
+    ): FutureData<D2TrackerEvent[]> {
+        if (chunked) {
+            return this.getRawSubstanceConsumptionDataByEventsIdsChunked(
+                orgUnitId,
+                substanceIds,
+                substanceIdsChunkSize
+            );
+        }
+        return Future.fromPromise(this.getRawSubstanceConsumptionDataByEventsIdsAsync(orgUnitId, substanceIds)).map(
             d2Events => d2Events
         );
     }
 
+    private getRawSubstanceConsumptionDataByEventsIdsChunked(
+        orgUnitId: Id,
+        substanceIds: Id[],
+        substanceIdsChunkSize: number
+    ): FutureData<D2TrackerEvent[]> {
+        const chunkedSubstanceIds = _(substanceIds).chunk(substanceIdsChunkSize).value();
+
+        return Future.sequential(
+            chunkedSubstanceIds.flatMap(substanceIdsChunk => {
+                const substanceIdsString = substanceIdsChunk.join(";");
+
+                // TODO: change pageSize to skipPaging:true when new version of d2-api
+                return apiToFuture(
+                    this.api.tracker.events.get({
+                        fields: eventFields,
+                        program: AMC_RAW_SUBSTANCE_CONSUMPTION_PROGRAM_ID,
+                        orgUnit: orgUnitId,
+                        event: substanceIdsString,
+                        pageSize: substanceIdsChunk.length,
+                    })
+                ).flatMap((eventsResponse: TrackerEventsResponse) => {
+                    return Future.success(eventsResponse.instances);
+                });
+            })
+        ).flatMap(listOfEvents => Future.success(_(listOfEvents).flatten().value()));
+    }
+
     private async getRawSubstanceConsumptionDataByEventsIdsAsync(
         orgUnit: Id,
         eventsIds: Id[]

diff --git a/src/data/repositories/data-entry/DataValuesDefaultRepository.ts b/src/data/repositories/data-entry/DataValuesDefaultRepository.ts
@@ -30,7 +30,7 @@ export class DataValuesDefaultRepository implements DataValuesRepository {
         ).flatMap(response => {
             return apiToFuture(this.api.system.waitFor(response.response.jobType, response.response.id)).map(result => {
                 if (result) {
-                    return {
+                    const importStatus: DataValuesSaveSummary = {
                         status: result.status,
                         description: result.description,
                         importCount: {
@@ -39,11 +39,12 @@ export class DataValuesDefaultRepository implements DataValuesRepository {
                             ignored: result.importCount.ignored,
                             deleted: result.importCount.deleted,
                         },
-                        conficts: result.conflicts,
+                        conflicts: result.conflicts,
                         importTime: new Date(response.response.created),
                     };
+                    return importStatus;
                 } else {
-                    return {
+                    const errorStatus: DataValuesSaveSummary = {
                         status: "ERROR",
                         description: "An unexpected error has ocurred saving data values",
                         importCount: {
@@ -52,9 +53,11 @@ export class DataValuesDefaultRepository implements DataValuesRepository {
                             ignored: 0,
                             deleted: 0,
                         },
-                        conficts: [],
+                        conflicts: [],
                         importTime: new Date(response.response.created),
                     };
+
+                    return errorStatus;
                 }
             });
         });

diff --git a/src/domain/entities/GlassModule.ts b/src/domain/entities/GlassModule.ts
@@ -12,6 +12,10 @@ interface ModuleUserGroups {
 }
 
 type DataSubmissionPeriodTypes = "YEARLY" | "QUARTERLY";
+export type ChunkSizes = {
+    productIds: number;
+    substanceIds: number;
+};
 
 export interface GlassModule {
     name: string;
@@ -41,6 +45,7 @@ export interface GlassModule {
     populateCurrentYearInHistory?: boolean;
     customDataColumns?: CustomDataColumns;
     lineLists?: LineListDetails[];
+    chunkSizes?: ChunkSizes;
 }
 
 export type LineListDetails = { id: Id; name?: string; programId: Id; programStageId?: Id };