diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 67c3adecb..035c2a822 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,10 +5,13 @@ on: jobs: unit-tests: name: Unit tests - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 + + - name: Install apt libraries + run: sudo apt install gettext -y - name: Setup Node uses: actions/setup-node@v1 @@ -31,7 +34,7 @@ jobs: restore-keys: | ${{ runner.os }}-yarn- - - name: Installing Dependencies + - name: Install dependencies run: yarn install --frozen-lockfile --silent - name: Install translations diff --git a/i18n/en.pot b/i18n/en.pot index 9a9d340b9..588428c08 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2021-12-13T07:26:20.684Z\n" -"PO-Revision-Date: 2021-12-13T07:26:20.684Z\n" +"POT-Creation-Date: 2021-12-30T06:26:16.031Z\n" +"PO-Revision-Date: 2021-12-30T06:26:16.031Z\n" msgid "" "THIS NEW RELEASE INCLUDES SHARING SETTINGS PER INSTANCES. FOR THIS VERSION " @@ -129,6 +129,9 @@ msgstr "" msgid "Event Program with Data Elements" msgstr "" +msgid "Program with Data Elements to Aggregated" +msgstr "" + msgid "Tracker Program with Program Stages" msgstr "" @@ -768,6 +771,15 @@ msgstr "" msgid "Invalid package" msgstr "" +msgid "Extending Dhis2 compatibility" +msgstr "" + +msgid "Dhis2 Compatibility extended successfully" +msgstr "" + +msgid "An error has ocurred extending Dhis2 compatibility" +msgstr "" + msgid "Publishing package to Store" msgstr "" @@ -839,6 +851,9 @@ msgstr "" msgid "Download as JSON" msgstr "" +msgid "Extend DHIS2 version compatibility" +msgstr "" + msgid "Publish to Store" msgstr "" @@ -893,6 +908,20 @@ msgstr "" msgid "Local" msgstr "" +msgid "" +"Extend DHIS2 version compatibility for module {{module}} and version " +"{{version}}" +msgstr "" + +msgid "Create package(s)" +msgstr "" + +msgid "Copy From (*)" +msgstr "" + +msgid "New DHIS2 version compatibility (*)" +msgstr "" + msgid "Back" msgstr "" @@ -1066,7 +1095,7 @@ msgstr "" msgid "Message" msgstr "" -msgid "Program Indicators" +msgid "Program Indicators / Program Data Elements" msgstr "" msgid "Aggregated" @@ -2022,6 +2051,9 @@ msgstr "" msgid "Finished Sync Rule {{name}}" msgstr "" +msgid "Program Indicators" +msgstr "" + msgid "never" msgstr "" diff --git a/i18n/es.po b/i18n/es.po index 08efe976c..4af13b8bf 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: i18next-conv\n" -"POT-Creation-Date: 2021-11-10T11:18:59.186Z\n" +"POT-Creation-Date: 2022-01-14T07:42:34.635Z\n" "PO-Revision-Date: 2020-07-10T06:53:30.625Z\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -129,6 +129,9 @@ msgstr "" msgid "Event Program with Data Elements" msgstr "" +msgid "Program with Data Elements to Aggregated" +msgstr "" + msgid "Tracker Program with Program Stages" msgstr "" @@ -769,6 +772,15 @@ msgstr "" msgid "Invalid package" msgstr "" +msgid "Extending Dhis2 compatibility" +msgstr "" + +msgid "Dhis2 Compatibility extended successfully" +msgstr "" + +msgid "An error has ocurred extending Dhis2 compatibility" +msgstr "" + msgid "Publishing package to Store" msgstr "" @@ -840,6 +852,9 @@ msgstr "" msgid "Download as JSON" msgstr "" +msgid "Extend DHIS2 version compatibility" +msgstr "" + msgid "Publish to Store" msgstr "" @@ -894,6 +909,20 @@ msgstr "" msgid "Local" msgstr "" +msgid "" +"Extend DHIS2 version compatibility for module {{module}} and version " +"{{version}}" +msgstr "" + +msgid "Create package(s)" +msgstr "" + +msgid "Copy From (*)" +msgstr "" + +msgid "New DHIS2 version compatibility (*)" +msgstr "" + msgid "Back" msgstr "" @@ -1069,7 +1098,7 @@ msgstr "" msgid "Message" msgstr "" -msgid "Program Indicators" +msgid "Program Indicators / Program Data Elements" msgstr "" msgid "Aggregated" @@ -2026,6 +2055,9 @@ msgstr "" msgid "Finished Sync Rule {{name}}" msgstr "" +msgid "Program Indicators" +msgstr "" + msgid "never" msgstr "" diff --git a/i18n/fr.po b/i18n/fr.po index 0acfceaf5..5337a5b62 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: i18next-conv\n" -"POT-Creation-Date: 2021-11-10T11:18:59.186Z\n" +"POT-Creation-Date: 2022-01-14T07:42:34.635Z\n" "PO-Revision-Date: 2020-07-10T06:53:30.625Z\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -129,6 +129,9 @@ msgstr "" msgid "Event Program with Data Elements" msgstr "" +msgid "Program with Data Elements to Aggregated" +msgstr "" + msgid "Tracker Program with Program Stages" msgstr "" @@ -769,6 +772,15 @@ msgstr "" msgid "Invalid package" msgstr "" +msgid "Extending Dhis2 compatibility" +msgstr "" + +msgid "Dhis2 Compatibility extended successfully" +msgstr "" + +msgid "An error has ocurred extending Dhis2 compatibility" +msgstr "" + msgid "Publishing package to Store" msgstr "" @@ -840,6 +852,9 @@ msgstr "" msgid "Download as JSON" msgstr "" +msgid "Extend DHIS2 version compatibility" +msgstr "" + msgid "Publish to Store" msgstr "" @@ -894,6 +909,20 @@ msgstr "" msgid "Local" msgstr "" +msgid "" +"Extend DHIS2 version compatibility for module {{module}} and version " +"{{version}}" +msgstr "" + +msgid "Create package(s)" +msgstr "" + +msgid "Copy From (*)" +msgstr "" + +msgid "New DHIS2 version compatibility (*)" +msgstr "" + msgid "Back" msgstr "" @@ -1069,7 +1098,7 @@ msgstr "" msgid "Message" msgstr "" -msgid "Program Indicators" +msgid "Program Indicators / Program Data Elements" msgstr "" msgid "Aggregated" @@ -2026,6 +2055,9 @@ msgstr "" msgid "Finished Sync Rule {{name}}" msgstr "" +msgid "Program Indicators" +msgstr "" + msgid "never" msgstr "" diff --git a/i18n/pt.po b/i18n/pt.po index 0acfceaf5..5337a5b62 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: i18next-conv\n" -"POT-Creation-Date: 2021-11-10T11:18:59.186Z\n" +"POT-Creation-Date: 2022-01-14T07:42:34.635Z\n" "PO-Revision-Date: 2020-07-10T06:53:30.625Z\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -129,6 +129,9 @@ msgstr "" msgid "Event Program with Data Elements" msgstr "" +msgid "Program with Data Elements to Aggregated" +msgstr "" + msgid "Tracker Program with Program Stages" msgstr "" @@ -769,6 +772,15 @@ msgstr "" msgid "Invalid package" msgstr "" +msgid "Extending Dhis2 compatibility" +msgstr "" + +msgid "Dhis2 Compatibility extended successfully" +msgstr "" + +msgid "An error has ocurred extending Dhis2 compatibility" +msgstr "" + msgid "Publishing package to Store" msgstr "" @@ -840,6 +852,9 @@ msgstr "" msgid "Download as JSON" msgstr "" +msgid "Extend DHIS2 version compatibility" +msgstr "" + msgid "Publish to Store" msgstr "" @@ -894,6 +909,20 @@ msgstr "" msgid "Local" msgstr "" +msgid "" +"Extend DHIS2 version compatibility for module {{module}} and version " +"{{version}}" +msgstr "" + +msgid "Create package(s)" +msgstr "" + +msgid "Copy From (*)" +msgstr "" + +msgid "New DHIS2 version compatibility (*)" +msgstr "" + msgid "Back" msgstr "" @@ -1069,7 +1098,7 @@ msgstr "" msgid "Message" msgstr "" -msgid "Program Indicators" +msgid "Program Indicators / Program Data Elements" msgstr "" msgid "Aggregated" @@ -2026,6 +2055,9 @@ msgstr "" msgid "Finished Sync Rule {{name}}" msgstr "" +msgid "Program Indicators" +msgstr "" + msgid "never" msgstr "" diff --git a/package.json b/package.json index 907c20ebf..3864a3439 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "metadata-synchronization", "description": "Advanced metadata & data synchronization utility", - "version": "2.14.0", + "version": "2.15.0", "license": "GPL-3.0", "author": "EyeSeeTea team", "homepage": ".", diff --git a/src/data/aggregated/AggregatedD2ApiRepository.ts b/src/data/aggregated/AggregatedD2ApiRepository.ts index 8e41d4768..78a428f5a 100644 --- a/src/data/aggregated/AggregatedD2ApiRepository.ts +++ b/src/data/aggregated/AggregatedD2ApiRepository.ts @@ -19,6 +19,7 @@ import { D2Api, DataValueSetsPostResponse } from "../../types/d2-api"; import { cache } from "../../utils/cache"; import { promiseMap } from "../../utils/common"; import { getD2APiFromInstance } from "../../utils/d2-utils"; +import mime from "mime-types"; export class AggregatedD2ApiRepository implements AggregatedRepository { private api: D2Api; @@ -330,6 +331,39 @@ export class AggregatedD2ApiRepository implements AggregatedRepository { } } + async getDataValueFile( + orgUnit: string, + period: string, + dataElement: string, + categoryOptionCombo: string, + value: string + ): Promise { + const blob = await this.api + .request({ + method: "get", + url: `/dataValues/files`, + responseDataType: "raw", + params: { + ou: orgUnit, + pe: period, + de: dataElement, + co: categoryOptionCombo, + value, + }, + }) + .getData(); + + if (!blob) throw Error("An error has ocurred retrieving the file resource of data value"); + + const fileResource = await this.api + .get<{ name: string; contentType: string }>(`/fileResources/${value}`) + .getData(); + + const fileName = fileResource?.name || `File.${mime.extension(fileResource.contentType)}`; + + return new File([blob], fileName, { type: fileResource.contentType }); + } + private cleanAggregatedImportResponse(importResult: DataValueSetsPostResponse): SynchronizationResult { const { status, description, importCount, conflicts } = importResult; diff --git a/src/data/events/EventsD2ApiRepository.ts b/src/data/events/EventsD2ApiRepository.ts index 069fa28ae..19a9a57ac 100644 --- a/src/data/events/EventsD2ApiRepository.ts +++ b/src/data/events/EventsD2ApiRepository.ts @@ -73,6 +73,8 @@ export class EventsD2ApiRepository implements EventsRepository { startDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined, endDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined, lastUpdated: lastUpdated ? moment(lastUpdated).format("YYYY-MM-DD") : undefined, + // @ts-ignore FIXME: Add property in d2-api + fields: ":all", }) .getData(); }; @@ -119,6 +121,8 @@ export class EventsD2ApiRepository implements EventsRepository { startDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined, endDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined, lastUpdated: lastUpdated ? moment(lastUpdated).toISOString() : undefined, + // @ts-ignore FIXME: Add property in d2-api + fields: ":all", }) .getData(); }; @@ -162,6 +166,8 @@ export class EventsD2ApiRepository implements EventsRepository { .getAll({ programStage, event: ids.join(";"), + // @ts-ignore FIXME: Add property in d2-api + fields: ":all", }) .getData(); result.push(...events); diff --git a/src/data/instance/InstanceFileD2Repository.ts b/src/data/instance/InstanceFileD2Repository.ts index 0f89b3d56..e5e48d8d9 100644 --- a/src/data/instance/InstanceFileD2Repository.ts +++ b/src/data/instance/InstanceFileD2Repository.ts @@ -1,7 +1,12 @@ +import { FileUploadParameters } from "@eyeseetea/d2-api/api/files"; import mime from "mime-types"; import { Instance } from "../../domain/instance/entities/Instance"; -import { FileId, InstanceFileRepository } from "../../domain/instance/repositories/InstanceFileRepository"; -import { D2Api } from "../../types/d2-api"; +import { + FileId, + FileResourceDomain, + InstanceFileRepository, +} from "../../domain/instance/repositories/InstanceFileRepository"; +import { D2Api, D2ApiResponse } from "../../types/d2-api"; import { getD2APiFromInstance } from "../../utils/d2-utils"; export class InstanceFileD2Repository implements InstanceFileRepository { @@ -23,18 +28,62 @@ export class InstanceFileD2Repository implements InstanceFileRepository { return this.blobToFile(response, `${documentName}.${mime.extension(response.type)}`); } - public async save(file: File): Promise { - const fileResourceId = await this.api.files - .saveFileResource({ - name: file.name, - data: file, + public async save(file: File, domain: FileResourceDomain = "DOCUMENT"): Promise { + if (domain === "DOCUMENT") { + const fileResourceId = await this.api.files + .saveFileResource({ + name: file.name, + data: file, + }) + .getData(); + + return fileResourceId; + } else { + const fileResourceId = await this.saveFileResource( + { + name: file.name, + data: file, + }, + domain + ).getData(); + + return fileResourceId; + } + } + + saveFileResource(params: Omit, domain: FileResourceDomain): D2ApiResponse { + const { name, data } = params; + + const formData = new FormData(); + formData.append("file", data, name); + formData.append("contentType", data.type); + formData.append("domain", domain); + + return this.api.apiConnection + .request({ + method: "post", + url: "/fileResources", + data: formData, + requestBodyType: "raw", }) - .getData(); + .map(({ data }) => { + if (!data.response || !data.response.fileResource || !data.response.fileResource.id) { + throw new Error("Unable to store file, couldn't find resource"); + } - return fileResourceId; + return data.response.fileResource.id; + }); } private blobToFile = (blob: Blob, fileName: string): File => { return new File([blob], fileName, { type: blob.type }); }; } + +interface PartialSaveResponse { + response?: { + fileResource?: { + id?: string; + }; + }; +} diff --git a/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts b/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts index 33a314798..45675b2df 100644 --- a/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts +++ b/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts @@ -9,12 +9,14 @@ import { startDhis } from "../../../../utils/dhisServer"; import { AggregatedD2ApiRepository } from "../../../aggregated/AggregatedD2ApiRepository"; import { ConfigAppRepository } from "../../../config/ConfigAppRepository"; import { InstanceD2ApiRepository } from "../../../instance/InstanceD2ApiRepository"; +import { InstanceFileD2Repository } from "../../../instance/InstanceFileD2Repository"; +import { MappingD2ApiRepository } from "../../../mapping/MappingD2ApiRepository"; import { TransformationD2ApiRepository } from "../../../transformations/TransformationD2ApiRepository"; import { MetadataD2ApiRepository } from "../../MetadataD2ApiRepository"; const repositoryFactory = buildRepositoryFactory(); -describe("Sync metadata", () => { +describe("Sync local instance mapped", () => { let local: Server; beforeAll(() => { @@ -36,7 +38,7 @@ describe("Sync metadata", () => { })); local.get("/metadata", async (_schema, request) => { - if (request.queryParams.filter === "id:in:[dataSet1]") + if (request.queryParams.filter === "id:in:[dataSet1]") { return { dataSets: [ { @@ -56,16 +58,25 @@ describe("Sync metadata", () => { }, ], }; - - if (request.queryParams.filter === "identifiable:eq:default") + } else if (request.queryParams.filter === "identifiable:eq:default") { return { categoryOptions: [{ id: "default1" }], categories: [{ id: "default2" }], categoryCombos: [{ id: "default3" }], categoryOptionCombos: [{ id: "default4" }], }; - - console.error("Unknown metadata request", request.queryParams); + } else if (request.queryParams.filter === "id:in:[id1]" && request.queryParams.fields === "id,valueType") { + return { + dataElements: [ + { + id: "id1", + valueType: "TEXT", + }, + ], + }; + } else { + console.error("Unknown metadata request", request.queryParams); + } }); local.get("/dataValueSets", async () => ({ @@ -95,8 +106,18 @@ describe("Sync metadata", () => { }, ]); - local.get("/dataStore/metadata-synchronization/instances-LOCAL", async () => ({ - metadataMapping: { + local.get("/dataStore/metadata-synchronization/instances-LOCAL", async () => ({})); + local.get("/dataStore/metadata-synchronization/mappings", async () => [ + { + id: "MAPPINGLOCAL", + owner: { + id: "LOCAL", + type: "instance", + }, + }, + ]); + local.get("/dataStore/metadata-synchronization/mappings-MAPPINGLOCAL", async () => ({ + mappingDictionary: { aggregatedDataElements: { id1: { mappedId: "id2", @@ -189,6 +210,8 @@ function buildRepositoryFactory() { repositoryFactory.bind(Repositories.MetadataRepository, MetadataD2ApiRepository); repositoryFactory.bind(Repositories.AggregatedRepository, AggregatedD2ApiRepository); repositoryFactory.bind(Repositories.TransformationRepository, TransformationD2ApiRepository); + repositoryFactory.bind(Repositories.MappingRepository, MappingD2ApiRepository); + repositoryFactory.bind(Repositories.InstanceFileRepository, InstanceFileD2Repository); return repositoryFactory; } diff --git a/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts b/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts index 0aab78efd..09059dae9 100644 --- a/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts +++ b/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts @@ -9,12 +9,14 @@ import { startDhis } from "../../../../utils/dhisServer"; import { AggregatedD2ApiRepository } from "../../../aggregated/AggregatedD2ApiRepository"; import { ConfigAppRepository } from "../../../config/ConfigAppRepository"; import { InstanceD2ApiRepository } from "../../../instance/InstanceD2ApiRepository"; +import { InstanceFileD2Repository } from "../../../instance/InstanceFileD2Repository"; +import { MappingD2ApiRepository } from "../../../mapping/MappingD2ApiRepository"; import { TransformationD2ApiRepository } from "../../../transformations/TransformationD2ApiRepository"; import { MetadataD2ApiRepository } from "../../MetadataD2ApiRepository"; const repositoryFactory = buildRepositoryFactory(); -describe("Sync metadata", () => { +describe("Sync aggregated", () => { let local: Server; let remote: Server; @@ -52,7 +54,7 @@ describe("Sync metadata", () => { })); local.get("/metadata", async (_schema, request) => { - if (request.queryParams.filter === "id:in:[dataSet1]") + if (request.queryParams.filter === "id:in:[dataSet1]") { return { dataSets: [ { @@ -72,16 +74,25 @@ describe("Sync metadata", () => { }, ], }; - - if (request.queryParams.filter === "identifiable:eq:default") + } else if (request.queryParams.filter === "identifiable:eq:default") { return { categoryOptions: [{ id: "default1" }], categories: [{ id: "default2" }], categoryCombos: [{ id: "default3" }], categoryOptionCombos: [{ id: "default4" }], }; - - console.error("Unknown metadata request", request.queryParams); + } else if (request.queryParams.filter === "id:in:[id1]" && request.queryParams.fields === "id,valueType") { + return { + dataElements: [ + { + id: "id1", + valueType: "TEXT", + }, + ], + }; + } else { + console.error("Unknown metadata request", request.queryParams); + } }); local.get("/dataValueSets", async () => ({ @@ -146,8 +157,18 @@ describe("Sync metadata", () => { ]); local.get("/dataStore/metadata-synchronization/instances-LOCAL", async () => ({})); - local.get("/dataStore/metadata-synchronization/instances-DESTINATION", async () => ({ - metadataMapping: { + local.get("/dataStore/metadata-synchronization/instances-DESTINATION", async () => ({})); + local.get("/dataStore/metadata-synchronization/mappings", async () => [ + { + id: "MAPPINGDESTINATION", + owner: { + id: "DESTINATION", + type: "instance", + }, + }, + ]); + local.get("/dataStore/metadata-synchronization/mappings-MAPPINGDESTINATION", async () => ({ + mappingDictionary: { aggregatedDataElements: { id1: { mappedId: "id2", @@ -291,6 +312,8 @@ function buildRepositoryFactory() { repositoryFactory.bind(Repositories.MetadataRepository, MetadataD2ApiRepository); repositoryFactory.bind(Repositories.AggregatedRepository, AggregatedD2ApiRepository); repositoryFactory.bind(Repositories.TransformationRepository, TransformationD2ApiRepository); + repositoryFactory.bind(Repositories.MappingRepository, MappingD2ApiRepository); + repositoryFactory.bind(Repositories.InstanceFileRepository, InstanceFileD2Repository); return repositoryFactory; } diff --git a/src/data/metadata/__tests__/integration/sync-events.spec.ts b/src/data/metadata/__tests__/integration/sync-events.spec.ts index 406f9cee2..f5fc7e99f 100644 --- a/src/data/metadata/__tests__/integration/sync-events.spec.ts +++ b/src/data/metadata/__tests__/integration/sync-events.spec.ts @@ -13,10 +13,11 @@ import { TEID2ApiRepository } from "../../../tracked-entity-instances/TEID2ApiRe import { InstanceD2ApiRepository } from "../../../instance/InstanceD2ApiRepository"; import { TransformationD2ApiRepository } from "../../../transformations/TransformationD2ApiRepository"; import { MetadataD2ApiRepository } from "../../MetadataD2ApiRepository"; +import { MappingD2ApiRepository } from "../../../mapping/MappingD2ApiRepository"; const repositoryFactory = buildRepositoryFactory(); -describe("Sync metadata", () => { +describe("Sync events", () => { let local: Server; let remote: Server; @@ -187,6 +188,7 @@ describe("Sync metadata", () => { local.get("/dataStore/metadata-synchronization/instances-LOCAL", async () => ({})); local.get("/dataStore/metadata-synchronization/instances-DESTINATION", async () => ({})); + local.get("/dataStore/metadata-synchronization/mappings", async () => []); local.get("/dataStore/metadata-synchronization/instances-LOCAL/metaData", async () => ({ created: "2021-03-30T01:59:59.191", @@ -328,6 +330,7 @@ function buildRepositoryFactory() { repositoryFactory.bind(Repositories.EventsRepository, EventsD2ApiRepository); repositoryFactory.bind(Repositories.TEIsRepository, TEID2ApiRepository); repositoryFactory.bind(Repositories.TransformationRepository, TransformationD2ApiRepository); + repositoryFactory.bind(Repositories.MappingRepository, MappingD2ApiRepository); return repositoryFactory; } diff --git a/src/data/migrations/tasks/08.remove-coc-inner-mappings.ts b/src/data/migrations/tasks/08.remove-coc-inner-mappings.ts index 93d7dec9e..2577255cc 100644 --- a/src/data/migrations/tasks/08.remove-coc-inner-mappings.ts +++ b/src/data/migrations/tasks/08.remove-coc-inner-mappings.ts @@ -75,10 +75,9 @@ export async function migrate(storage: AppStorage, _debug: Debug, params: Migrat if (oldMetadataMapping?.programDataElements || oldMetadataMapping?.aggregatedDataElements) { const programDataElements = cleanProgramDataElements(oldMetadataMapping.programDataElements); - const aggregatedDataElements = oldMetadataMapping.aggregatedDataElements ? await cleanAggregatedDataElements( - d2Api, - oldMetadataMapping.aggregatedDataElements - ) : undefined; + const aggregatedDataElements = oldMetadataMapping.aggregatedDataElements + ? await cleanAggregatedDataElements(d2Api, oldMetadataMapping.aggregatedDataElements) + : undefined; const metadataMapping = { ...oldMetadataMapping, diff --git a/src/data/tracked-entity-instances/TEID2ApiRepository.ts b/src/data/tracked-entity-instances/TEID2ApiRepository.ts index 2d2c2cd77..05ab0efd2 100644 --- a/src/data/tracked-entity-instances/TEID2ApiRepository.ts +++ b/src/data/tracked-entity-instances/TEID2ApiRepository.ts @@ -13,7 +13,7 @@ import { import { cleanOrgUnitPaths } from "../../domain/synchronization/utils"; import { TEIsPackage } from "../../domain/tracked-entity-instances/entities/TEIsPackage"; import { TrackedEntityInstance } from "../../domain/tracked-entity-instances/entities/TrackedEntityInstance"; -import { TEIRepository } from "../../domain/tracked-entity-instances/repositories/TEIRepository"; +import { TEIRepository, TEIsResponse } from "../../domain/tracked-entity-instances/repositories/TEIRepository"; import { D2Api } from "../../types/d2-api"; import { getD2APiFromInstance } from "../../utils/d2-utils"; @@ -28,13 +28,26 @@ export class TEID2ApiRepository implements TEIRepository { constructor(private instance: Instance) { this.api = getD2APiFromInstance(instance); } - async getTEIs(params: DataSynchronizationParams, program: string): Promise { + async getTEIs( + params: DataSynchronizationParams, + program: string, + page: number, + pageSize: number + ): Promise { const { period, orgUnitPaths = [] } = params; const { startDate, endDate } = buildPeriodFromParams(params); const orgUnits = cleanOrgUnitPaths(orgUnitPaths); - if (orgUnits.length === 0) return []; + if (orgUnits.length === 0) + return { + trackedEntityInstances: [], + pager: { + pageSize, + total: 0, + page, + }, + }; const result = await this.api .get("/trackedEntityInstances", { @@ -43,10 +56,13 @@ export class TEID2ApiRepository implements TEIRepository { fields: this.fields, programStartDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined, programEndDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined, + totalPages: true, + page, + pageSize, }) .getData(); - return result.trackedEntityInstances; + return { ...result }; } async getTEIsById(params: DataSynchronizationParams, ids: string[]): Promise { @@ -147,7 +163,3 @@ interface TEIsPostResponse { }[]; }; } - -interface TEIsResponse { - trackedEntityInstances: TrackedEntityInstance[]; -} diff --git a/src/data/transformations/TransformationD2ApiRepository.ts b/src/data/transformations/TransformationD2ApiRepository.ts index 8d6d08a1e..6eba0110c 100644 --- a/src/data/transformations/TransformationD2ApiRepository.ts +++ b/src/data/transformations/TransformationD2ApiRepository.ts @@ -15,10 +15,11 @@ export class TransformationD2ApiRepository implements TransformationRepository { public mapPackageTo( destination: number, payload: Input, - transformations: Transformation[] = [] + transformations: Transformation[] = [], + origin?: number ): Output { const transformationstoApply = _.orderBy(transformations, ["apiVersion"]).filter( - ({ apiVersion }) => apiVersion <= destination && apiVersion > API_VERSION + ({ apiVersion }) => apiVersion <= destination && apiVersion > (origin || API_VERSION) ); if (transformationstoApply.length > 0) { @@ -40,10 +41,11 @@ export class TransformationD2ApiRepository implements TransformationRepository { public mapPackageFrom( origin: number, payload: Input, - transformations: Transformation[] = [] + transformations: Transformation[] = [], + destination?: number ): Output { const transformationstoApply = _.orderBy(transformations, ["apiVersion"], ["desc"]).filter( - ({ apiVersion }) => apiVersion <= origin && apiVersion > API_VERSION + ({ apiVersion }) => apiVersion <= origin && apiVersion > (destination || API_VERSION) ); if (transformationstoApply.length > 0) { diff --git a/src/domain/aggregated/mapper/AggregatedPayloadMapper.ts b/src/domain/aggregated/mapper/AggregatedPayloadMapper.ts index 4a416472c..683c5412d 100644 --- a/src/domain/aggregated/mapper/AggregatedPayloadMapper.ts +++ b/src/domain/aggregated/mapper/AggregatedPayloadMapper.ts @@ -51,8 +51,10 @@ export class AggregatedPayloadMapper implements PayloadMapper { const { organisationUnits = {}, aggregatedDataElements = {} } = globalMapping; const { mapping: innerMapping = {} } = aggregatedDataElements[dataElement] ?? {}; + const dataElementId = dataElement.replace(".", "-"); + const mappedOrgUnit = organisationUnits[orgUnit]?.mappedId ?? orgUnit; - const mappedDataElement = aggregatedDataElements[dataElement]?.mappedId ?? dataElement; + const mappedDataElement = aggregatedDataElements[dataElementId]?.mappedId ?? dataElement; const mappedValue = mapOptionValue(value, [innerMapping, globalMapping]); const mappedComment = mapOptionValue(comment, [innerMapping, globalMapping]); const mappedCategory = diff --git a/src/domain/aggregated/mapper/__tests__/AggregatedPayloadMapper.spec.ts b/src/domain/aggregated/mapper/__tests__/AggregatedPayloadMapper.spec.ts index e982b54d0..bb108f8e4 100644 --- a/src/domain/aggregated/mapper/__tests__/AggregatedPayloadMapper.spec.ts +++ b/src/domain/aggregated/mapper/__tests__/AggregatedPayloadMapper.spec.ts @@ -8,6 +8,7 @@ import dataValues from "./data/data-values/dataValues.json"; import dataValuesIndicator from "./data/data-values/dataValues_indicator.json"; import dataValuesCommentOption from "./data/data-values/dataValues_comment_option.json"; import dataValuesProgramIndicator from "./data/data-values/dataValues_program_indicator.json"; +import dataValuesProgramdataElement from "./data/data-values/dataValues_program_data_element.json"; import orgUnitsMapping from "./data/mapping/mapping_orgUnits.json"; import dataElementsMapping from "./data/mapping/mapping_dataelements.json"; @@ -24,6 +25,7 @@ import disabledGlobalOptionMapping from "./data/mapping/mapping_disabled_global_ import disabledOrgUnitsMapping from "./data/mapping/mapping_disabled_orgUnits.json"; import indicatorDataElementMapping from "./data/mapping/mapping_indicator_dataelement.json"; import programIndicatorDataElementMapping from "./data/mapping/mapping_program_indicator_dataelement.json"; +import programDataElementToAggregatedMapping from "./data/mapping/mapping_program_data_element_aggregated.json"; import dataValuesWithoutMapping from "./data/expected/dataValues_without_mapping.json"; import dataValuesOrgUnitsMapping from "./data/expected/dataValues_orgunits_mapping.json"; @@ -40,6 +42,7 @@ import dataValuesDisabledOptionMapping from "./data/expected/dataValues_disabled import dataValuesIndicatorDataElementMapping from "./data/expected/dataValues_indicator_dataelement_mapping.json"; import dataValuesCommentMapping from "./data/expected/dataValues_comment_option_mapping.json"; import dataValuesProgramIndicatorDataElementMapping from "./data/expected/dataValues_program_indicator_de_mapping.json"; +import dataValuesProgramDataElementToAggregatedMapping from "./data/expected/dataValues_program_DE_Aggregated_mapping.json"; describe("AggreggatedPayloadMapper", () => { it("should return the expected payload if mapping is empty", async () => { @@ -161,6 +164,13 @@ describe("AggreggatedPayloadMapper", () => { expect(mappedPayload).toEqual(dataValuesProgramIndicatorDataElementMapping); }); + it("should return the payload with mapped data element if mapping contain program dataelement to aggregated mapping", async () => { + const aggregatedMapper = createAggregatedPayloadMapper(programDataElementToAggregatedMapping); + + const mappedPayload = await aggregatedMapper.map(dataValuesProgramdataElement); + + expect(mappedPayload).toEqual(dataValuesProgramDataElementToAggregatedMapping); + }); }); function createAggregatedPayloadMapper(mapping: MetadataMappingDictionary): AggregatedPayloadMapper { diff --git a/src/domain/aggregated/mapper/__tests__/data/data-values/dataValues_program_data_element.json b/src/domain/aggregated/mapper/__tests__/data/data-values/dataValues_program_data_element.json new file mode 100644 index 000000000..94b2811e8 --- /dev/null +++ b/src/domain/aggregated/mapper/__tests__/data/data-values/dataValues_program_data_element.json @@ -0,0 +1,10 @@ +{ + "dataValues": [ + { + "dataElement": "AdPw08GkY15.MA6VeqyUe4q", + "period": "2021", + "orgUnit": "BWW8zaLISW4", + "value": "12.0" + } + ] +} diff --git a/src/domain/aggregated/mapper/__tests__/data/expected/dataValues_program_DE_Aggregated_mapping.json b/src/domain/aggregated/mapper/__tests__/data/expected/dataValues_program_DE_Aggregated_mapping.json new file mode 100644 index 000000000..d4b3dd917 --- /dev/null +++ b/src/domain/aggregated/mapper/__tests__/data/expected/dataValues_program_DE_Aggregated_mapping.json @@ -0,0 +1,10 @@ +{ + "dataValues": [ + { + "dataElement": "F6yeCU8jIbq", + "period": "2021", + "orgUnit": "BWW8zaLISW4", + "value": "12.0" + } + ] +} diff --git a/src/domain/aggregated/mapper/__tests__/data/mapping/mapping_program_data_element_aggregated.json b/src/domain/aggregated/mapper/__tests__/data/mapping/mapping_program_data_element_aggregated.json new file mode 100644 index 000000000..547ccc055 --- /dev/null +++ b/src/domain/aggregated/mapper/__tests__/data/mapping/mapping_program_data_element_aggregated.json @@ -0,0 +1,11 @@ +{ + "aggregatedDataElements": { + "AdPw08GkY15-MA6VeqyUe4q": { + "global": false, + "mapping": {}, + "mappedId": "F6yeCU8jIbq", + "conflicts": false, + "mappedName": "Aggregated DE 1" + } + } +} diff --git a/src/domain/aggregated/repositories/AggregatedRepository.ts b/src/domain/aggregated/repositories/AggregatedRepository.ts index 4e5fb141b..9788a3fcd 100644 --- a/src/domain/aggregated/repositories/AggregatedRepository.ts +++ b/src/domain/aggregated/repositories/AggregatedRepository.ts @@ -34,4 +34,12 @@ export interface AggregatedRepository { save(data: AggregatedPackage, additionalParams?: DataImportParams): Promise; delete(data: AggregatedPackage): Promise; + + getDataValueFile( + orgUnit: string, + period: string, + dataElement: string, + value: string, + categoryOptionCombo: string + ): Promise; } diff --git a/src/domain/aggregated/usecases/AggregatedSyncUseCase.ts b/src/domain/aggregated/usecases/AggregatedSyncUseCase.ts index d1eefc859..5d9d5e454 100644 --- a/src/domain/aggregated/usecases/AggregatedSyncUseCase.ts +++ b/src/domain/aggregated/usecases/AggregatedSyncUseCase.ts @@ -14,13 +14,15 @@ import { SynchronizationResult } from "../../reports/entities/SynchronizationRes import { GenericSyncUseCase } from "../../synchronization/usecases/GenericSyncUseCase"; import { buildMetadataDictionary } from "../../synchronization/utils"; import { AggregatedPackage } from "../entities/AggregatedPackage"; +import { DataValue } from "../entities/DataValue"; import { createAggregatedPayloadMapper } from "../mapper/AggregatedPayloadMapperFactory"; import { getMinimumParents } from "../utils"; +import { promiseMap } from "../../../utils/common"; export class AggregatedSyncUseCase extends GenericSyncUseCase { public readonly type = "aggregated"; public readonly fields = - "id,dataElements[id,name],dataSetElements[:all,dataElement[id,name]],dataElementGroups[id,dataElements[id,name]],name"; + "id,dataElements[id,name,valueType],dataSetElements[:all,dataElement[id,name,valueType]],dataElementGroups[id,dataElements[id,name,valueType]],name"; public buildPayload = memoize(async (remoteInstance?: Instance) => { const { dataParams: { enableAggregation = false } = {} } = this.builder; @@ -134,7 +136,10 @@ export class AggregatedSyncUseCase extends GenericSyncUseCase { public async postPayload(instance: Instance): Promise { const { dataParams = {} } = this.builder; - const originalPayload = await this.buildPayload(); + const previousOriginalPayload = await this.buildPayload(); + + const originalPayload = await this.manageDataElementWithFileType(previousOriginalPayload, instance); + const mappedPayload = await this.mapPayload(instance, originalPayload); const existingPayload = dataParams.ignoreDuplicateExistingValues @@ -156,6 +161,43 @@ export class AggregatedSyncUseCase extends GenericSyncUseCase { return [{ ...syncResult, origin: origin.toPublicObject(), payload }]; } + private async manageDataElementWithFileType( + payload: { dataValues: DataValue[] }, + remoteInstance: Instance + ): Promise<{ dataValues: DataValue[] }> { + const metadataRepository = await this.getMetadataRepository(); + const { dataElements = [] } = await metadataRepository.getMetadataByIds( + payload.dataValues.map(dv => dv.dataElement).flat(), + "id,valueType" + ); + + const dataElementFileTypes = dataElements.filter(de => de.valueType === "FILE_RESOURCE").map(de => de.id); + + const aggregatedRepository = await this.getAggregatedRepository(); + const fileRemoteRepository = await this.getInstanceFileRepository(remoteInstance); + + const dataValues = await promiseMap(payload.dataValues, async dataValue => { + const isFileType = dataElementFileTypes.includes(dataValue.dataElement); + + if (isFileType) { + const file = await aggregatedRepository.getDataValueFile( + dataValue.orgUnit, + dataValue.period, + dataValue.dataElement, + dataValue.categoryOptionCombo || "", + dataValue.value + ); + + const destinationFileId = await fileRemoteRepository.save(file, "DATA_VALUE"); + return { ...dataValue, value: destinationFileId }; + } else { + return dataValue; + } + }); + + return { dataValues }; + } + public async buildDataStats() { const metadataPackage = await this.extractMetadata(); const dictionary = buildMetadataDictionary(metadataPackage); diff --git a/src/domain/events/usecases/EventsSyncUseCase.ts b/src/domain/events/usecases/EventsSyncUseCase.ts index 407e7a3d6..8078b0519 100644 --- a/src/domain/events/usecases/EventsSyncUseCase.ts +++ b/src/domain/events/usecases/EventsSyncUseCase.ts @@ -61,10 +61,26 @@ export class EventsSyncUseCase extends GenericSyncUseCase { programs?.map(({ programIndicators }: Partial) => programIndicators?.map(({ id }) => id) ?? []) ); + // Due to a limitation in the analytics endpoint, it's not possible request with dx dimension by program stage and data element + // only program and data element is allowed. For this reason to avoid duplicate dimension error, we remove data element duplicated + // using the uniq function. Duplicate data elements is possible for tracker programs here if two stages has the same data element + // Jira-issue: https://jira.dhis2.org/browse/DHIS2-12382 + + const dataElementsByProgram = _(programs) + .flatMap(({ id, programStages }: Partial) => + _.flatMap( + programStages, + ({ programStageDataElements }) => + programStageDataElements.map(({ dataElement }) => `${id}.${dataElement.id}`) ?? [] + ) + ) + .uniq() + .value(); + const { dataValues: candidateDataValues = [] } = enableAggregation ? await aggregatedRepository.getAnalytics({ dataParams, - dimensionIds: [...directIndicators, ...indicatorsByProgram], + dimensionIds: [...directIndicators, ...indicatorsByProgram, ...dataElementsByProgram], includeCategories: false, }) : {}; diff --git a/src/domain/instance/repositories/InstanceFileRepository.ts b/src/domain/instance/repositories/InstanceFileRepository.ts index 0782b03cb..2b0d790c8 100644 --- a/src/domain/instance/repositories/InstanceFileRepository.ts +++ b/src/domain/instance/repositories/InstanceFileRepository.ts @@ -6,7 +6,9 @@ export interface InstanceFileRepositoryConstructor { export type FileId = string; +export type FileResourceDomain = "DOCUMENT" | "DATA_VALUE"; + export interface InstanceFileRepository { getById(fileId: FileId): Promise; - save(file: File): Promise; + save(file: File, domain?: FileResourceDomain): Promise; } diff --git a/src/domain/packages/usecases/ExtendsPackagesFromPackageUseCase.ts b/src/domain/packages/usecases/ExtendsPackagesFromPackageUseCase.ts new file mode 100644 index 000000000..dd6dbdfe1 --- /dev/null +++ b/src/domain/packages/usecases/ExtendsPackagesFromPackageUseCase.ts @@ -0,0 +1,55 @@ +import { generateUid } from "d2/uid"; +import { Namespace } from "../../../data/storage/Namespaces"; +import { metadataTransformations } from "../../../data/transformations/PackageTransformations"; +import { getMajorVersion } from "../../../utils/d2-utils"; +import { UseCase } from "../../common/entities/UseCase"; +import { RepositoryFactory } from "../../common/factories/RepositoryFactory"; +import { Instance } from "../../instance/entities/Instance"; +import { BasePackage, Package } from "../entities/Package"; + +export class ExtendsPackagesFromPackageUseCase implements UseCase { + constructor(private repositoryFactory: RepositoryFactory, private localInstance: Instance) {} + + public async execute(packageSourceId: string, dhisVersions: string[]): Promise { + const storageClient = await this.repositoryFactory.configRepository(this.localInstance).getStorageClient(); + const transformationRepository = this.repositoryFactory.transformationRepository(); + + const packageData = await storageClient.getObjectInCollection(Namespace.PACKAGES, packageSourceId); + + const pkg = Package.build(packageData); + + const user = await this.repositoryFactory.userRepository(this.localInstance).getCurrent(); + + for (const dhisVersion of dhisVersions) { + const originApiVersion = getMajorVersion(pkg.dhisVersion); + const destinationApiVersion = getMajorVersion(dhisVersion); + + const versionedPayload = + destinationApiVersion > originApiVersion + ? transformationRepository.mapPackageTo( + destinationApiVersion, + pkg.contents, + metadataTransformations, + originApiVersion + ) + : transformationRepository.mapPackageFrom( + originApiVersion, + pkg.contents, + metadataTransformations, + destinationApiVersion + ); + + const newPackage = pkg.update({ + id: generateUid(), + dhisVersion, + created: new Date(), + lastUpdated: new Date(), + lastUpdatedBy: user, + user: user, + contents: versionedPayload, + }); + + await storageClient.saveObjectInCollection(Namespace.PACKAGES, newPackage); + } + } +} diff --git a/src/domain/rules/entities/SynchronizationRule.ts b/src/domain/rules/entities/SynchronizationRule.ts index a98bf9306..f0dab8b48 100644 --- a/src/domain/rules/entities/SynchronizationRule.ts +++ b/src/domain/rules/entities/SynchronizationRule.ts @@ -636,14 +636,15 @@ export class SynchronizationRule { } : null, ]), - dataSyncEventsOrTeis: _.compact([ + dataSyncEventsTeisOrAggregation: _.compact([ this.type === "events" && !this.dataSyncAllEvents && this.dataSyncEvents.length === 0 && - this.dataSyncTeis.length === 0 + this.dataSyncTeis.length === 0 && + !this.dataSyncEnableAggregation ? { key: "cannot_be_empty", - namespace: { element: "event or TEI" }, + namespace: { element: "event, TEI or enable data aggregation" }, } : null, ]), diff --git a/src/domain/synchronization/usecases/GenericSyncUseCase.ts b/src/domain/synchronization/usecases/GenericSyncUseCase.ts index 380929e9e..c39f1ad66 100644 --- a/src/domain/synchronization/usecases/GenericSyncUseCase.ts +++ b/src/domain/synchronization/usecases/GenericSyncUseCase.ts @@ -118,7 +118,7 @@ export abstract class GenericSyncUseCase { public async getMapping(instance: Instance): Promise { const { originInstance: originInstanceId } = this.builder; - const mappingRepository = await this.getMappingRepository(); + const mappingRepository = await this.getMappingRepository(instance.id === "LOCAL" ? instance : undefined); // If sync is LOCAL -> REMOTE, use the destination instance mapping if (originInstanceId === "LOCAL") { diff --git a/src/domain/tracked-entity-instances/repositories/TEIRepository.ts b/src/domain/tracked-entity-instances/repositories/TEIRepository.ts index cbcb83363..59e762adb 100644 --- a/src/domain/tracked-entity-instances/repositories/TEIRepository.ts +++ b/src/domain/tracked-entity-instances/repositories/TEIRepository.ts @@ -9,8 +9,17 @@ export interface TEIRepositoryConstructor { } export interface TEIRepository { - getTEIs(params: DataSynchronizationParams, program: string): Promise; + getTEIs(params: DataSynchronizationParams, program: string, page: number, pageSize: number): Promise; getTEIsById(params: DataSynchronizationParams, ids: string[]): Promise; save(data: TEIsPackage, additionalParams: DataImportParams | undefined): Promise; } + +export interface TEIsResponse { + trackedEntityInstances: TrackedEntityInstance[]; + pager: { + pageSize: number; + total: number; + page: number; + }; +} diff --git a/src/domain/tracked-entity-instances/usecases/ListTEIsUseCase.ts b/src/domain/tracked-entity-instances/usecases/ListTEIsUseCase.ts index d3b8fdfbe..9d12c6def 100644 --- a/src/domain/tracked-entity-instances/usecases/ListTEIsUseCase.ts +++ b/src/domain/tracked-entity-instances/usecases/ListTEIsUseCase.ts @@ -2,7 +2,7 @@ import { DataSynchronizationParams } from "../../aggregated/entities/DataSynchro import { UseCase } from "../../common/entities/UseCase"; import { RepositoryFactory } from "../../common/factories/RepositoryFactory"; import { Instance } from "../../instance/entities/Instance"; -import { TrackedEntityInstance } from "../entities/TrackedEntityInstance"; +import { TEIsResponse } from "../repositories/TEIRepository"; export class ListTEIsUseCase implements UseCase { constructor(private repositoryFactory: RepositoryFactory, protected localInstance: Instance) {} @@ -10,8 +10,10 @@ export class ListTEIsUseCase implements UseCase { public async execute( params: DataSynchronizationParams, programs: string, - instance: Instance - ): Promise { - return this.repositoryFactory.teisRepository(instance).getTEIs(params, programs); + instance: Instance, + page: number, + pageSize: number + ): Promise { + return this.repositoryFactory.teisRepository(instance).getTEIs(params, programs, page, pageSize); } } diff --git a/src/domain/transformations/repositories/TransformationRepository.ts b/src/domain/transformations/repositories/TransformationRepository.ts index 7dc49b36e..5886ded51 100644 --- a/src/domain/transformations/repositories/TransformationRepository.ts +++ b/src/domain/transformations/repositories/TransformationRepository.ts @@ -7,14 +7,16 @@ export interface TransformationRepositoryConstructor { export interface TransformationRepository { mapPackageTo( - version: number, + destination: number, payload: Input, - transformations: Transformation[] + transformations: Transformation[], + origin?: number ): Output; mapPackageFrom( - version: number, + origin: number, payload: Input, - transformations: Transformation[] + transformations: Transformation[], + destination?: number ): Output; } diff --git a/src/models/dhis/mapping.ts b/src/models/dhis/mapping.ts index d71c70bd7..7135e508d 100644 --- a/src/models/dhis/mapping.ts +++ b/src/models/dhis/mapping.ts @@ -176,6 +176,48 @@ export class EventProgramWithDataElementsModel extends EventProgramModel { }; } +export class ProgramWithDataElementsToAggregatedModel extends EventProgramWithDataElementsModel { + protected static metadataType = "programWithDataElementsToAggregated"; + protected static modelName = i18n.t("Program with Data Elements to Aggregated"); + protected static childrenKeys = ["dataElements"]; + protected static fields = programFieldsWithDataElements; + protected static modelFilters: any = {}; + + // Due to a limitation in the analytics endpoint, it's not possible request with dx dimension by program stage and data element + // only program and data element is allowed. For this reason the complex id for program data element to aggregated data element is + // program-dataElement + // Jira-issue: https://jira.dhis2.org/browse/DHIS2-12382 + + protected static modelTransform = ( + objects: SelectedPick[] + ) => { + return objects.map(program => ({ + ...program, + dataElements: _.flatten( + program.programStages?.map(({ displayName, programStageDataElements }) => + programStageDataElements + .filter(({ dataElement }) => !!dataElement) + .map(({ dataElement }) => ({ + ...dataElement, + id: `${program.id}-${dataElement.id}`, + parentId: `${program.id}`, + model: ProgramDataElementToAggregatedModel, + displayName: + program.programStages.length > 1 + ? `[${displayName}] ${dataElement.displayName}` + : dataElement.displayName, + })) + ) ?? [] + ), + })); + }; +} + +export class ProgramDataElementToAggregatedModel extends ProgramDataElementModel { + protected static parentMappingType = "programWithDataElementsToAggregated"; + protected static mappingType = "aggregatedDataElements"; +} + export class EventProgramWithProgramStagesModel extends TrackerProgramModel { protected static metadataType = "programWithProgramStages"; protected static modelName = i18n.t("Tracker Program with Program Stages"); diff --git a/src/presentation/CompositionRoot.ts b/src/presentation/CompositionRoot.ts index 457a46e33..b43a603d9 100644 --- a/src/presentation/CompositionRoot.ts +++ b/src/presentation/CompositionRoot.ts @@ -76,6 +76,7 @@ import { CreatePackageUseCase } from "../domain/packages/usecases/CreatePackageU import { DeletePackageUseCase } from "../domain/packages/usecases/DeletePackageUseCase"; import { DiffPackageUseCase } from "../domain/packages/usecases/DiffPackageUseCase"; import { DownloadPackageUseCase } from "../domain/packages/usecases/DownloadPackageUseCase"; +import { ExtendsPackagesFromPackageUseCase } from "../domain/packages/usecases/ExtendsPackagesFromPackageUseCase"; import { GetPackageUseCase } from "../domain/packages/usecases/GetPackageUseCase"; import { GetStorePackageUseCase } from "../domain/packages/usecases/GetStorePackageUseCase"; import { ImportPackageUseCase } from "../domain/packages/usecases/ImportPackageUseCase"; @@ -239,6 +240,7 @@ export class CompositionRoot { publish: new PublishStorePackageUseCase(this.repositoryFactory, this.localInstance), diff: new DiffPackageUseCase(this, this.repositoryFactory, this.localInstance), import: new ImportPackageUseCase(this.repositoryFactory, this.localInstance), + extend: new ExtendsPackagesFromPackageUseCase(this.repositoryFactory, this.localInstance), }); } diff --git a/src/presentation/react/core/components/package-list-table/PackageListTable.tsx b/src/presentation/react/core/components/package-list-table/PackageListTable.tsx index 66bb03d5b..15b0bb78f 100644 --- a/src/presentation/react/core/components/package-list-table/PackageListTable.tsx +++ b/src/presentation/react/core/components/package-list-table/PackageListTable.tsx @@ -34,10 +34,12 @@ import { DiffPackages, PackagesDiffDialog } from "../packages-diff-dialog/Packag import { groupPackageByModuleAndVersion as groupPackagesByModuleAndVersion, InstallStatus, + isModuleItem, isPackageItem, PackageItem, PackageModuleItem, } from "./PackageModuleItem"; +import { PackagesExtendCompatibilityDialog } from "../packages-extend_compatibility/PackagesExtendCompatibilityDialog"; interface PackagesListTableProps extends ModulePackageListPageProps { isImportDialog?: boolean; @@ -86,12 +88,29 @@ export const PackagesListTable: React.FC = ({ const [toImportWizard, setToImportWizard] = useState([]); + const [packagesToExtendCompatibility, setPackagesToExtendCompatibility] = useState( + undefined + ); + const isRemoteInstance = !!remoteInstance; useEffect(() => { compositionRoot.modules.list(globalAdmin, remoteInstance, true).then(setModules); }, [compositionRoot, globalAdmin, remoteInstance]); + const rowsFiltered = useMemo(() => { + setLoadingTable(false); + + const packageItems = rows.filter( + row => + (row.module.id === moduleFilter || !moduleFilter) && + (row.dhisVersion === dhis2VersionFilter || !dhis2VersionFilter) && + (row.installStatus === installStatusFilter || !installStatusFilter) + ); + + return groupPackagesByModuleAndVersion(packageItems); + }, [moduleFilter, rows, dhis2VersionFilter, installStatusFilter]); + const updateSelection = useCallback( (selection: TableSelection[]) => { updateStateSelection(selection); @@ -134,6 +153,43 @@ export const PackagesListTable: React.FC = ({ [compositionRoot, remoteInstance, snackbar, remoteStore] ); + const openExtendPackageCompatibilityDialog = useCallback( + async (ids: string[]) => { + const id = _.first(ids); + if (!id) return; + + const parent = rowsFiltered.find(parent => parent.id === id); + + setPackagesToExtendCompatibility(parent?.packages); + }, + [rowsFiltered] + ); + + const closeExtendPackageCompatibilityDialog = useCallback( + () => setPackagesToExtendCompatibility(undefined), + [setPackagesToExtendCompatibility] + ); + + const extendsPackagesFromPackageUseCase = useCallback( + (packageId: string, dhis2Versions: string[]) => { + loading.show(true, i18n.t("Extending Dhis2 compatibility")); + compositionRoot.packages + .extend(packageId, dhis2Versions) + .then(() => { + snackbar.success(i18n.t("Dhis2 Compatibility extended successfully")); + setResetKey(Math.random()); + loading.reset(); + setPackagesToExtendCompatibility(undefined); + }) + .catch(() => { + snackbar.error(i18n.t("An error has ocurred extending Dhis2 compatibility")); + loading.reset(); + setPackagesToExtendCompatibility(undefined); + }); + }, + [compositionRoot, snackbar, loading] + ); + const publishPackage = useCallback( async (ids: string[]) => { loading.show(true, i18n.t("Publishing package to Store")); @@ -468,6 +524,20 @@ export const PackagesListTable: React.FC = ({ icon: cloud_download, isActive: (rows: PackageModuleItem[]) => _.every(rows, row => isPackageItem(row)), }, + { + name: "extendCompatibility", + text: i18n.t("Extend DHIS2 version compatibility"), + multiple: false, + onClick: openExtendPackageCompatibilityDialog, + icon: extension, + isActive: (rows: PackageModuleItem[]) => + _.every(rows, row => isModuleItem(row)) && + !isImportDialog && + presentation === "app" && + !isRemoteInstance && + !remoteStore && + appConfigurator, + }, { name: "publish", text: i18n.t("Publish to Store"), @@ -568,6 +638,7 @@ export const PackagesListTable: React.FC = ({ selectedIds, generateModule, modules, + openExtendPackageCompatibilityDialog, ] ); @@ -658,19 +729,6 @@ export const PackagesListTable: React.FC = ({ installStatusFilter, ]); - const rowsFiltered = useMemo(() => { - setLoadingTable(false); - - const packageItems = rows.filter( - row => - (row.module.id === moduleFilter || !moduleFilter) && - (row.dhisVersion === dhis2VersionFilter || !dhis2VersionFilter) && - (row.installStatus === installStatusFilter || !installStatusFilter) - ); - - return groupPackagesByModuleAndVersion(packageItems); - }, [moduleFilter, rows, dhis2VersionFilter, installStatusFilter]); - const handleOpenSyncSummaryFromDialog = (syncReport: SynchronizationReport) => { setOpenImportPackageDialog(false); setToImportWizard([]); @@ -804,6 +862,14 @@ export const PackagesListTable: React.FC = ({ openSyncSummary={handleOpenSyncSummaryFromDialog} /> )} + + {packagesToExtendCompatibility && ( + + )} ); }; diff --git a/src/presentation/react/core/components/package-list-table/PackageModuleItem.ts b/src/presentation/react/core/components/package-list-table/PackageModuleItem.ts index 7fd76326c..4447a7a0b 100644 --- a/src/presentation/react/core/components/package-list-table/PackageModuleItem.ts +++ b/src/presentation/react/core/components/package-list-table/PackageModuleItem.ts @@ -1,3 +1,4 @@ +import _ from "lodash"; import { BasePackage } from "../../../../../domain/packages/entities/Package"; import { FlattenUnion } from "../../../../../utils/flatten-union"; @@ -22,8 +23,14 @@ export const isPackageItem = (item: PackageModuleItem): item is PackageItem => { return (item as PackageItem).module !== undefined; }; +export const isModuleItem = (item: PackageModuleItem): item is PackageItem => { + return (item as PackageItem).module === undefined; +}; + export const groupPackageByModuleAndVersion = (packages: PackageItem[]) => { - return packages.reduce((acc, item) => { + const sortedPackages = _(packages).sortBy("dhisVersion").value(); + + return sortedPackages.reduce((acc, item) => { const parentKey = `${item.module.id}-${item.version}`; const parent = acc.find(parent => parent.id === parentKey); diff --git a/src/presentation/react/core/components/packages-extend_compatibility/PackagesExtendCompatibilityDialog.tsx b/src/presentation/react/core/components/packages-extend_compatibility/PackagesExtendCompatibilityDialog.tsx new file mode 100644 index 000000000..838fa9400 --- /dev/null +++ b/src/presentation/react/core/components/packages-extend_compatibility/PackagesExtendCompatibilityDialog.tsx @@ -0,0 +1,105 @@ +import { ConfirmationDialog } from "@eyeseetea/d2-ui-components"; +import { makeStyles, TextField } from "@material-ui/core"; +import { Autocomplete } from "@material-ui/lab"; +import React, { useMemo, useState, useEffect, useCallback } from "react"; +import i18n from "../../../../../locales"; +import { Dropdown, DropdownOption } from "../dropdown/Dropdown"; +import { PackageItem } from "../package-list-table/PackageModuleItem"; + +export interface PackagesExtendCompatibilityDialogProps { + onClose(): void; + onSave(packakeId: string, dhis2Versions: string[]): void; + packages: PackageItem[]; +} + +const dhis2Versions = ["2.30", "2.31", "2.32", "2.33", "2.34", "2.35", "2.36", "2.37"]; + +export const PackagesExtendCompatibilityDialog: React.FC = ({ + onSave, + onClose, + packages, +}) => { + const classes = useStyles(); + + const [existedDhis2Versions, SetExistedDhis2Versions] = useState([]); + const [selectedExistedDhis2Version, setSelectedExistedDhis2Version] = useState(""); + + const [newDhis2Versions, SetNewDhis2Versions] = useState([]); + const [selectedNewDhis2Versions, setSelectedNewDhis2Versions] = useState([]); + + useEffect(() => { + const dhis2VersionsInPackages = packages.map(pkg => pkg.dhisVersion); + SetNewDhis2Versions(dhis2Versions.filter(d2version => !dhis2VersionsInPackages.includes(d2version))); + SetExistedDhis2Versions( + dhis2VersionsInPackages.map(dhis2Version => ({ id: dhis2Version, name: dhis2Version })) + ); + }, [packages]); + + const title = useMemo(() => { + const firstPackage = packages[0]; + + return { + module: firstPackage.module.name, + version: firstPackage.version, + }; + }, [packages]); + + const handleSave = useCallback(() => { + if (!onSave) return; + + const selectedPkg = packages.find(pkg => pkg.dhisVersion === selectedExistedDhis2Version); + + if (!selectedPkg) return; + + onSave(selectedPkg.id, selectedNewDhis2Versions); + }, [onSave, selectedExistedDhis2Version, selectedNewDhis2Versions, packages]); + + return ( + + + + + setSelectedNewDhis2Versions(value)} + renderTags={(values: string[]) => values.sort().join(", ")} + renderInput={params => ( + + )} + /> + + + ); +}; + +const useStyles = makeStyles({ + row: { + marginTop: 10, + marginLeft: 10, + }, +}); diff --git a/src/presentation/react/core/components/sync-summary/SyncSummary.tsx b/src/presentation/react/core/components/sync-summary/SyncSummary.tsx index 04754dc16..e6c71bbb9 100644 --- a/src/presentation/react/core/components/sync-summary/SyncSummary.tsx +++ b/src/presentation/react/core/components/sync-summary/SyncSummary.tsx @@ -160,7 +160,7 @@ const buildMessageTable = (messages: ErrorMessage[]) => { const getTypeName = (reportType: SynchronizationResultType, syncType: string) => { switch (reportType) { case "aggregated": - return syncType === "events" ? i18n.t("Program Indicators") : i18n.t("Aggregated"); + return syncType === "events" ? i18n.t("Program Indicators / Program Data Elements") : i18n.t("Aggregated"); case "events": return i18n.t("Events"); case "trackedEntityInstances": diff --git a/src/presentation/react/core/components/sync-wizard/Steps.ts b/src/presentation/react/core/components/sync-wizard/Steps.ts index 9dad8133b..cefb6aa8f 100644 --- a/src/presentation/react/core/components/sync-wizard/Steps.ts +++ b/src/presentation/react/core/components/sync-wizard/Steps.ts @@ -196,6 +196,7 @@ export const eventsSteps: SyncWizardStep[] = [ }, { ...commonSteps.aggregation, + validationKeys: ["dataSyncAggregation", "dataSyncEventsTeisOrAggregation"], warning: i18n.t( "If aggregation is enabled, the synchronization will use the Analytics endpoint and group data by organisation units children and the chosen time periods. Program indicators are only included during a synchronization if aggregation is enabled." ), diff --git a/src/presentation/react/core/components/sync-wizard/data/TEIsSelectionStep.tsx b/src/presentation/react/core/components/sync-wizard/data/TEIsSelectionStep.tsx index f320ffc05..ba2a37d35 100644 --- a/src/presentation/react/core/components/sync-wizard/data/TEIsSelectionStep.tsx +++ b/src/presentation/react/core/components/sync-wizard/data/TEIsSelectionStep.tsx @@ -3,6 +3,7 @@ import { ObjectsTable, ObjectsTableDetailField, TableColumn, + TablePagination, TableState, useSnackbar, } from "@eyeseetea/d2-ui-components"; @@ -34,6 +35,9 @@ export default function TEIsSelectionStep({ syncRule, onChange }: SyncWizardStep const [programs, setPrograms] = useState([]); const [programFilter, setProgramFilter] = useState(""); const [error, setError] = useState(); + const [paginationFilters, setPaginationFilters] = useState({ page: 1, pageSize: 25 }); + const [pager, setPager] = useState>({}); + const [rowsLoading, setRowsLoading] = useState(false); useEffect(() => { const sync = compositionRoot.sync.events(memoizedSyncRule.toBuilder()); @@ -50,6 +54,7 @@ export default function TEIsSelectionStep({ syncRule, onChange }: SyncWizardStep useEffect(() => { if (programFilter) { + setRowsLoading(true); compositionRoot.instances.getById(syncRule.originInstance).then(result => { result.match({ error: () => snackbar.error(i18n.t("Invalid origin instance")), @@ -58,12 +63,22 @@ export default function TEIsSelectionStep({ syncRule, onChange }: SyncWizardStep .list( { ...memoizedSyncRule.dataParams, - allEvents: true, }, programFilter, - instance + instance, + paginationFilters.page, + paginationFilters.pageSize ) - .then(teis => setRows(teis.map(tei => ({ ...tei, id: tei.trackedEntityInstance })))) + .then(teisResponse => { + setPager(teisResponse.pager); + setRows( + teisResponse.trackedEntityInstances.map(tei => ({ + ...tei, + id: tei.trackedEntityInstance, + })) + ); + setRowsLoading(false); + }) .catch(setError); }, }); @@ -71,12 +86,20 @@ export default function TEIsSelectionStep({ syncRule, onChange }: SyncWizardStep } else { setRows([]); } - }, [compositionRoot, programFilter, memoizedSyncRule, syncRule.originInstance, snackbar]); + }, [ + compositionRoot, + programFilter, + memoizedSyncRule.dataParams, + syncRule.originInstance, + snackbar, + paginationFilters, + ]); const handleTableChange = useCallback( (tableState: TableState) => { - const { selection } = tableState; + const { selection, pagination } = tableState; onChange(syncRule.updateDataSyncTEIs(selection.map(({ id }) => id))); + setPaginationFilters({ page: pagination.page, pageSize: pagination.pageSize }); }, [onChange, syncRule] ); @@ -189,7 +212,7 @@ export default function TEIsSelectionStep({ syncRule, onChange }: SyncWizardStep rows={rows} - loading={rows === undefined} + loading={rowsLoading} columns={columns} details={details} actions={actions} @@ -197,6 +220,7 @@ export default function TEIsSelectionStep({ syncRule, onChange }: SyncWizardStep onChange={handleTableChange} selection={syncRule.dataSyncTeis?.map(id => ({ id })) ?? []} filterComponents={filterComponents} + pagination={pager} /> diff --git a/src/presentation/react/core/components/sync-wizard/metadata/MetadataIncludeExcludeStep.tsx b/src/presentation/react/core/components/sync-wizard/metadata/MetadataIncludeExcludeStep.tsx index 5274dd0b3..81f239c94 100644 --- a/src/presentation/react/core/components/sync-wizard/metadata/MetadataIncludeExcludeStep.tsx +++ b/src/presentation/react/core/components/sync-wizard/metadata/MetadataIncludeExcludeStep.tsx @@ -32,6 +32,7 @@ const MetadataIncludeExcludeStep: React.FC = ({ syncRule, o const [modelSelectItems, setModelSelectItems] = useState([]); const [models, setModels] = useState([]); + const [pendingApplyUseDefaultChange, setPendingApplyUseDefaultChange] = useState(false); const [selectedType, setSelectedType] = useState(""); const { compositionRoot } = useAppContext(); @@ -57,11 +58,19 @@ const MetadataIncludeExcludeStep: React.FC = ({ syncRule, o setModels(models); setModelSelectItems(options); + + if (pendingApplyUseDefaultChange) { + onChange( + syncRule.useDefaultIncludeExclude + ? syncRule.markToUseDefaultIncludeExclude() + : syncRule.markToNotUseDefaultIncludeExclude(models) + ); + } }); }, }); }); - }, [compositionRoot, api, syncRule, snackbar]); + }, [compositionRoot, api, syncRule, snackbar, pendingApplyUseDefaultChange, onChange]); const { includeRules = [], excludeRules = [] } = syncRule.metadataIncludeExcludeRules[selectedType] || {}; const allRules = [...includeRules, ...excludeRules]; @@ -71,6 +80,9 @@ const MetadataIncludeExcludeStep: React.FC = ({ syncRule, o })); const changeUseDefaultIncludeExclude = (useDefault: boolean) => { + if (models.length === 0) { + setPendingApplyUseDefaultChange(true); + } onChange( useDefault ? syncRule.markToUseDefaultIncludeExclude() : syncRule.markToNotUseDefaultIncludeExclude(models) ); diff --git a/src/presentation/webapp/core/pages/instance-mapping/InstanceMappingPage.tsx b/src/presentation/webapp/core/pages/instance-mapping/InstanceMappingPage.tsx index a483424b8..10defd897 100644 --- a/src/presentation/webapp/core/pages/instance-mapping/InstanceMappingPage.tsx +++ b/src/presentation/webapp/core/pages/instance-mapping/InstanceMappingPage.tsx @@ -20,6 +20,7 @@ import { IndicatorMappedModel, OrganisationUnitMappedModel, ProgramIndicatorMappedModel, + ProgramWithDataElementsToAggregatedModel, RelationshipTypeMappedModel, TrackedEntityAttributeToDEMappedModel, TrackedEntityAttributeToTEIMappedModel, @@ -40,6 +41,7 @@ const config = { models: [ EventProgramWithDataElementsModel, EventProgramWithProgramStagesMappedModel, + ProgramWithDataElementsToAggregatedModel, EventProgramWithIndicatorsModel, ProgramIndicatorMappedModel, RelationshipTypeMappedModel,