[Feature] Data Quality admin report

i18n/en.pot

@@ -35,6 +35,15 @@ msgstr ""
 msgid "Help"
 msgstr ""
 
+msgid "Data quality"
+msgstr ""
+
+msgid "Indicators"
+msgstr ""
+
+msgid "ProgramIndicators"
+msgstr ""
+
 msgid "Metadata Admin Report"
 msgstr ""
 

i18n/es.po

@@ -35,6 +35,15 @@ msgstr "Volver"
 msgid "Help"
 msgstr "Ayuda"
 
+msgid "Data quality"
+msgstr ""
+
+msgid "Indicators"
+msgstr ""
+
+msgid "ProgramIndicators"
+msgstr ""
+
 msgid "Metadata Admin Report"
 msgstr ""
 

package.json

@@ -31,6 +31,7 @@
         "d2-manifest": "1.0.0",
         "dotenv-flow": "3.2.0",
         "font-awesome": "4.7.0",
+        "i18n-locales": "^0.0.5",
         "lodash": "4.17.21",
         "node-html-parser": "5.1.0",
         "postcss-rtl": "1.7.3",

src/compositionRoot.ts

@@ -1,3 +1,4 @@
+import { DataQualityDefaultRepository } from "./data/DataQualityDefaultRepository";
 import { Dhis2ConfigRepository } from "./data/Dhis2ConfigRepository";
 import { Dhis2OrgUnitsRepository } from "./data/Dhis2OrgUnitsRepository";
 import { NHWADataApprovalDefaultRepository } from "./data/NHWADataApprovalDefaultRepository";
@@ -7,6 +8,7 @@ import { GetWIDPAdminDefaultUseCase } from "./domain/admin/usecases/GetWIDPAdmin
 import { SaveWIDPAdminDefaultCsvUseCase } from "./domain/admin/usecases/SaveWIDPAdminDefaultCsvUseCase";
 import { GetConfig } from "./domain/common/usecases/GetConfig";
 import { GetOrgUnitsUseCase } from "./domain/common/usecases/GetOrgUnitsUseCase";
+import { GetDataQualityDefaultUseCase } from "./domain/data-quality/usecases/GetDataQualityDefaultUseCase";
 import { UpdateStatusUseCase } from "./domain/nhwa-approval-status/usecases/CompleteDataSetsUseCase";
 import { GetApprovalColumnsUseCase } from "./domain/nhwa-approval-status/usecases/GetApprovalColumnsUseCase";
 import { GetDataSetsUseCase } from "./domain/nhwa-approval-status/usecases/GetDataSetsUseCase";
@@ -21,9 +23,18 @@ export function getCompositionRoot(api: D2Api) {
     const dataCommentsRepository = new NHWADataCommentsDefaultRepository(api);
     const dataApprovalRepository = new NHWADataApprovalDefaultRepository(api);
     const widpAdminDefaultRepository = new WIDPAdminDefaultRepository(api);
+    const dataQualityRepository = new DataQualityDefaultRepository(api);
     const orgUnitsRepository = new Dhis2OrgUnitsRepository(api);
 
     return {
+        dataQuality: getExecute({
+            getIndicators: new GetDataQualityDefaultUseCase({
+                indicators: true
+            }, dataQualityRepository),
+            getProgramIndicators: new GetProgramIndicatorstUseCase(dataQualityRepository),
+            saveIndicators: new SaveIndicatorsUseCase(dataQualityRepository),
+            saveProgramIndicators: new SaveProgramIndicatorsUseCase(dataQualityRepository),
+        }),
         admin: getExecute({
             get: new GetWIDPAdminDefaultUseCase(widpAdminDefaultRepository),
             save: new SaveWIDPAdminDefaultCsvUseCase(widpAdminDefaultRepository),

src/data/DataQualityDefaultRepository.ts

@@ -0,0 +1,199 @@
+import { D2Api } from "../types/d2-api";
+import { CsvWriterDataSource } from "./CsvWriterCsvDataSource";
+import { downloadFile } from "./utils/download-file";
+import { CsvData } from "../data/CsvDataSource";
+import { MetadataObject } from "../domain/common/entities/MetadataObject";
+import _ from "lodash";
+import { Indicator } from "../domain/common/entities/Indicator";
+import { ProgramIndicator } from "../domain/common/entities/ProgramIndicator";
+import { IndicatorsByExpression } from "../domain/common/entities/IndicatorsByExpression";
+import { DataQualityRepository } from "../domain/data-quality/repositories/DataQualityRepository";
+
+const filter = "filter=lastUpdated:gt:"
+
+export class DataQualityDefaultRepository implements DataQualityRepository {
+    constructor(private api: D2Api) { }
+
+    async getIndicators(startDate?: string): Promise<Indicator[]> {
+
+        const indicatorsResult: any = await this.api.metadata.d2Api
+            .get(
+                "/indicators.json?paging=false&fields=*" + startDate
+            )
+            .getData();
+
+        return indicatorsResult
+    }
+
+    async getIndicatorByExpression(lastUpdated: string): Promise<IndicatorsByExpression> {
+        const d2api = this.api
+        const list: IndicatorsByExpression = {
+            right: [],
+            wrongDenominator: [],
+            wrongNumerator: [],
+            wrongFilter: []
+        };
+        const startDate = lastUpdated ? lastUpdated : "&startDate=1970-01-01";
+        const indicatorsResult: any = await this.getIndicators(startDate)
+
+
+
+        _.map(indicatorsResult, function (indicator) {
+            const expresionResult: any = d2api.expressions.validate("indicator", indicator.numerator
+            )
+                .getData()
+
+            if (expresionResult["message"] != "Valid") {
+                list.wrongNumerator.push(indicator)
+            } else {
+                const expresionResult: any = d2api.expressions.validate("indicator", indicator.denominator
+                )
+                    .getData()
+                if (expresionResult["filter"] != "Valid") {
+                    list.wrongDenominator.push(indicator)
+                }
+                else {
+                    //todo maybe the filter needs other key?
+                    const expresionResult: any = d2api.expressions.validate("indicator", indicator.filter
+                    )
+                        .getData()
+
+                    if (expresionResult["filter"] != "Valid") {
+                        list.wrongFilter.push(indicator)
+                    } else {
+                        list.right.push(indicator)
+
+                    }
+                }
+            }
+        })
+        return list;
+
+        /*         return Promise.all(_.map(indicatorsResult, function (indicator) {
+                    const expresionResult: any = d2api.metadata.d2Api
+                        .post(
+                            "/indicators/expression/description", indicator.numerator
+                        )
+                        .getData()
+
+                    if (expresionResult["message"] != "Valid") {
+                        list.wrongNumerator.push(indicator)
+                    } else {
+                        const expresionResult: any = d2api.metadata.d2Api
+                            .post(
+                                "/indicators/expression/description", indicator.denominator
+                            )
+                            .getData()
+                        if (expresionResult["filter"] != "Valid") {
+                            list.wrongDenominator.push(indicator)
+                        }
+                        else {
+                            const expresionResult: any = d2api.metadata.d2Api
+                                .post(
+                                    "/indicators/expression/description", indicator.filter
+                                )
+                                .getData()
+
+                            if (expresionResult["filter"] != "Valid") {
+                                list.wrongFilter.push(indicator)
+                            } else {
+                                list.right.push(indicator)
+
+                            }
+                        }
+                    }
+                })); */
+    }
+
+
+    async getProgramIndicators(startDate?: string): Promise<Indicator[]> {
+
+        const indicatorsResult: any = await this.api.metadata.d2Api
+            .get(
+                "/programIndicators.json?ields=id,name,shortName,numerator,denominator,filter,expression,lastUpdatedBy[displayName],user[displayName],created,lastUpdated,userGroupAccesses,userAccess,publicAccess&paging=false" + startDate
+            )
+            .getData();
+
+        return indicatorsResult
+    }
+
+
+    async getProgramIndicatorsByExpression(lastUpdated: string): Promise<ProgramIndicator[]> {
+
+        const startDate = lastUpdated ? lastUpdated : "&startDate=1970-01-01";
+        const programIndicatorsResult: any = await this.getIndicators(startDate)
+        const programIndicators: ProgramIndicator[] = programIndicatorsResult
+        const d2api = this.api
+        const programIndicatorsByExpression = await Promise.all(_.map(programIndicators, function (programIndicator) {
+            const expresionResult: any = d2api.expressions.validate("program-indicator-formula", programIndicator.expression
+            )
+                .getData()
+
+            if (expresionResult["message"] != "Valid") {
+                wrongExpression: programIndicator
+            } else {
+                const expresionResult: any = d2api.expressions.validate("program-indicator-filter", programIndicator.filter
+                )
+                    .getData()
+                if (expresionResult["filter"] != "Valid") {
+                    wrongFilter: programIndicator
+                }
+                else {
+                    correct: programIndicator
+                }
+            }
+        }));
+        return programIndicatorsByExpression;
+    }
+    async saveLastUpdated(key: string, date: string): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+    async saveIndicators(key: string, indicators: Indicator[]): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+    async saveProgramIndicators(key: string, programIndicators: ProgramIndicator[]): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+
+
+    async save(filename: string, metadataObjects: MetadataObject[]): Promise<void> {
+        const headers = csvFields.map(field => ({ id: field, text: field }));
+        const rows = metadataObjects.map(
+            (metadataObject): MetadataRow => ({
+                metadataType: metadataObject.metadataType,
+                id: metadataObject.Id,
+                name: metadataObject.name,
+                publicAccess: metadataObject.publicAccess,
+                userGroupAccess: metadataObject.userGroupAccess || "-",
+                userAccess: metadataObject.userAccess || "-",
+                createdBy: metadataObject.createdBy || "-",
+                lastUpdatedBy: metadataObject.lastUpdatedBy || "-",
+                created: metadataObject.created || "-",
+                lastUpdated: metadataObject.lastUpdated || "-",
+            })
+        );
+
+        const csvDataSource = new CsvWriterDataSource();
+        const csvData: CsvData<CsvField> = { headers, rows };
+        const csvContents = csvDataSource.toString(csvData);
+
+        await downloadFile(csvContents, filename, "text/csv");
+    }
+}
+
+const csvFields = [
+    "metadataType",
+    "id",
+    "name",
+    "publicAccess",
+    "userGroupAccess",
+    "userAccess",
+    "createdBy",
+    "lastUpdatedBy",
+    "created",
+    "lastUpdated",
+] as const;
+
+type CsvField = typeof csvFields[number];
+
+type MetadataRow = Record<CsvField, string>;

src/domain/common/entities/Indicator.ts

@@ -0,0 +1,23 @@
+import { Id } from "./Base";
+
+export interface Indicator {
+    id: Id;
+    name: string;
+    shortName: string;
+    metadataType: string;
+    numerator: string;
+    nominator_error: string;
+    denominator_error: string;
+    filter_error: string;
+    expression: string;
+    type: string;
+    denominator: string;
+    filter: string;
+    publicAccess: string;
+    createdBy: string;
+    lastUpdatedBy: string;
+    userGroupAccess: string;
+    userAccess: string;
+    created: string;
+    lastUpdated: string;
+}

src/domain/common/entities/IndicatorsByExpression.ts

@@ -0,0 +1,8 @@
+import { Indicator } from "./Indicator";
+
+export interface IndicatorsByExpression {
+    right: Indicator[],
+    wrongNumerator: Indicator[],
+    wrongDenominator: Indicator[],
+    wrongFilter: Indicator[],
+}

src/domain/common/entities/ProgramIndicator.ts

@@ -0,0 +1,16 @@
+import { Id } from "./Base";
+
+export interface ProgramIndicator {
+    Id: Id;
+    name: string;
+    metadataType: string;
+    publicAccess: string;
+    expression: string
+    filter: string;
+    createdBy?: string;
+    lastUpdatedBy?: string;
+    userGroupAccess?: string;
+    userAccess?: string;
+    lastUpdated?: string;
+    created?: string;
+}

src/domain/common/entities/ProgramIndicatorsByExpression.ts

@@ -0,0 +1,7 @@
+import { ProgramIndicator } from "./ProgramIndicator";
+
+export interface ProgramIndicatorsByExpression {
+    right: ProgramIndicator[],
+    wrongExpression: ProgramIndicator[],
+    wrongFilter: ProgramIndicator[],
+}

src/domain/data-quality/repositories/DataQualityRepository.ts

@@ -0,0 +1,25 @@
+import { IndicatorsByExpression } from "../../common/entities/IndicatorsByExpression";
+import { ProgramIndicatorsByExpression } from "../../common/entities/ProgramIndicatorsByExpression";
+
+export interface DataQualityRepository {
+    getIndicatorByExpressions(lastUpdated: string): Promise<IndicatorsByExpression>;
+    getProgramIndicatorByExpressions(lastUpdated: string): Promise<ProgramIndicatorsByExpression>;
+    getLastUpdated(key: string): string;
+    saveIndicators(lastUpdated: string, indicators: IndicatorsByExpression): Promise<void>;
+    saveProgramIndicators(lastUpdated: string, programIndicators: ProgramIndicatorsByExpression): Promise<void>;
+}
+
+
+export interface DataQualityRepositoryGetOptions {
+    indicators: boolean;
+    programIndicators: boolean;
+    indicatorLastUpdated: string;
+    programIndicatorsLastUpdated: string;
+}
+
+export interface DataQualityRepositorySaveOptions {
+    indicators?: IndicatorsByExpression,
+    programIndicators?: ProgramIndicatorsByExpression
+    indicatorLastUpdated: string;
+    programIndicatorsLastUpdated: string;
+}

src/domain/data-quality/usecases/GetDataQualityDefaultUseCase.ts

@@ -0,0 +1,16 @@
+import { IndicatorsByExpression } from "../../common/entities/IndicatorsByExpression";
+import { ProgramIndicatorsByExpression } from "../../common/entities/ProgramIndicatorsByExpression";
+import { DataQualityRepository, DataQualityRepositoryGetOptions } from "../repositories/DataQualityRepository";
+
+export class GetDataQualityDefaultUseCase {
+    constructor(private dataQualityRepository: DataQualityRepository) { }
+
+    execute(options: DataQualityRepositoryGetOptions): Promise<IndicatorsByExpression | ProgramIndicatorsByExpression> {
+        // FUTURE: Return a Future-like instead, to allow better error handling and cancellation.
+        if (options.indicators) {
+            return this.dataQualityRepository.getIndicatorByExpressions(this.dataQualityRepository.getLastUpdated("indicators"));
+        } else {
+            return this.dataQualityRepository.getProgramIndicatorByExpressions(this.dataQualityRepository.getLastUpdated("programIndicators"));
+        }
+    }
+}

src/domain/data-quality/usecases/SaveDataQualityDefaultCsvUseCase.ts

@@ -0,0 +1,16 @@
+
+import { IndicatorsByExpression } from "../../common/entities/IndicatorsByExpression";
+import { ProgramIndicatorsByExpression } from "../../common/entities/ProgramIndicatorsByExpression";
+import { DataQualityRepository, DataQualityRepositorySaveOptions } from "../repositories/DataQualityRepository";
+
+export class SaveDataQualityDefaultCsvUseCase {
+    constructor(private metadataRepository: DataQualityRepository) { }
+
+    async execute(options: DataQualityRepositorySaveOptions): Promise<void> {
+        if (options.indicators) {
+            this.metadataRepository.saveIndicators(options.indicatorLastUpdated, options.indicators);
+        } else if (options.programIndicators) {
+            this.metadataRepository.saveProgramIndicators(options.programIndicatorsLastUpdated, options.programIndicators);
+        }
+    }
+}

src/webapp/components/dropdown/Dropdown.tsx

@@ -1,7 +1,7 @@
 import React from "react";
 import { MenuItem, Select } from "@material-ui/core";
 import DropdownForm from "./DropdownForm";
-import i18n from "../../../locales";
+import i18n from "@eyeseetea/d2-ui-components/locales";
 
 type Value = string;