Feature/admin dataquality

README.md

@@ -36,8 +36,8 @@ Create an standard report:
 
 ```
 $ yarn build-report # Creates dist/index.html
-$ yarn build-<key>-metadata -u 'user:pass' http://dhis2-server.org # Creates dist/metadata.json (key is a particular report group, e.g. nhwa)
-$ yarn post-<key>-metadata -u 'user:pass' http://dhis2-server.org # Posts dist/metadata.json (key is a particular report group, e.g. nhwa)
+$ yarn build-<key>-metadata -u 'user:pass' --url http://dhis2-server.org # Creates dist/metadata.json (key is a particular report group, e.g. nhwa)
+$ yarn post-<key>-metadata -u 'user:pass' --url http://dhis2-server.org # Posts dist/metadata.json (key is a particular report group, e.g. nhwa)
 ```
 
 Create an standalone DHIS2 webapp app:

i18n/en.pot

@@ -5,8 +5,8 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1)\n"
-"POT-Creation-Date: 2022-01-10T08:43:46.954Z\n"
-"PO-Revision-Date: 2022-01-10T08:43:46.954Z\n"
+"POT-Creation-Date: 2022-02-02T18:22:48.596Z\n"
+"PO-Revision-Date: 2022-02-02T18:22:48.596Z\n"
 
 msgid "<No value>"
 msgstr ""
@@ -74,15 +74,48 @@ msgstr ""
 msgid "created"
 msgstr ""
 
-msgid "NHWA Data Approval Status Report"
+msgid "Data quality"
 msgstr ""
 
-msgid "Data set"
+msgid "Indicators"
+msgstr ""
+
+msgid "ProgramIndicators"
+msgstr ""
+
+msgid "denominator"
+msgstr ""
+
+msgid "Valid Denominator"
+msgstr ""
+
+msgid "numerator"
+msgstr ""
+
+msgid "Valid Numerator"
+msgstr ""
+
+msgid "expression"
+msgstr ""
+
+msgid "Valid Expression"
+msgstr ""
+
+msgid "filter"
+msgstr ""
+
+msgid "Valid Filter"
+msgstr ""
+
+msgid "NHWA Data Approval Status Report"
 msgstr ""
 
 msgid "Period"
 msgstr ""
 
+msgid "Data set"
+msgstr ""
+
 msgid "Attribute"
 msgstr ""
 

i18n/es.po

@@ -1,7 +1,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: i18next-conv\n"
-"POT-Creation-Date: 2022-01-10T08:43:46.954Z\n"
+"POT-Creation-Date: 2022-02-02T18:22:48.596Z\n"
 "PO-Revision-Date: 2018-10-25T09:02:35.143Z\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -74,15 +74,48 @@ msgstr ""
 msgid "created"
 msgstr ""
 
-msgid "NHWA Data Approval Status Report"
+msgid "Data quality"
 msgstr ""
 
-msgid "Data set"
+msgid "Indicators"
+msgstr ""
+
+msgid "ProgramIndicators"
+msgstr ""
+
+msgid "denominator"
+msgstr ""
+
+msgid "Valid Denominator"
+msgstr ""
+
+msgid "numerator"
+msgstr ""
+
+msgid "Valid Numerator"
+msgstr ""
+
+msgid "expression"
+msgstr ""
+
+msgid "Valid Expression"
+msgstr ""
+
+msgid "filter"
+msgstr ""
+
+msgid "Valid Filter"
+msgstr ""
+
+msgid "NHWA Data Approval Status Report"
 msgstr ""
 
 msgid "Period"
 msgstr ""
 
+msgid "Data set"
+msgstr ""
+
 msgid "Attribute"
 msgstr ""
 

src/compositionRoot.ts

@@ -1,3 +1,4 @@
+import { DataQualityDefaultRepository } from "./data/DataQualityDefaultRepository";
 import { Dhis2ConfigRepository } from "./data/Dhis2ConfigRepository";
 import { Dhis2OrgUnitsRepository } from "./data/Dhis2OrgUnitsRepository";
 import { NHWADataApprovalDefaultRepository } from "./data/NHWADataApprovalDefaultRepository";
@@ -7,6 +8,8 @@ import { GetWIDPAdminDefaultUseCase } from "./domain/admin/usecases/GetWIDPAdmin
 import { SaveWIDPAdminDefaultCsvUseCase } from "./domain/admin/usecases/SaveWIDPAdminDefaultCsvUseCase";
 import { GetConfig } from "./domain/common/usecases/GetConfig";
 import { GetOrgUnitsUseCase } from "./domain/common/usecases/GetOrgUnitsUseCase";
+import { GetDataQualityDefaultUseCase } from "./domain/data-quality/usecases/GetDataQualityDefaultUseCase";
+import { SaveDataQualityDefaultCsvUseCase } from "./domain/data-quality/usecases/SaveDataQualityDefaultCsvUseCase";
 import { UpdateStatusUseCase } from "./domain/nhwa-approval-status/usecases/CompleteDataSetsUseCase";
 import { GetApprovalColumnsUseCase } from "./domain/nhwa-approval-status/usecases/GetApprovalColumnsUseCase";
 import { GetDataSetsUseCase } from "./domain/nhwa-approval-status/usecases/GetDataSetsUseCase";
@@ -21,9 +24,15 @@ export function getCompositionRoot(api: D2Api) {
     const dataCommentsRepository = new NHWADataCommentsDefaultRepository(api);
     const dataApprovalRepository = new NHWADataApprovalDefaultRepository(api);
     const widpAdminDefaultRepository = new WIDPAdminDefaultRepository(api);
+    const dataQualityRepository = new DataQualityDefaultRepository(api);
     const orgUnitsRepository = new Dhis2OrgUnitsRepository(api);
 
     return {
+        dataQuality: getExecute({
+            getValidations: new GetDataQualityDefaultUseCase(dataQualityRepository, true),
+            reloadValidations: new GetDataQualityDefaultUseCase(dataQualityRepository, false),
+            exportToCsv: new SaveDataQualityDefaultCsvUseCase(dataQualityRepository),
+        }),
         admin: getExecute({
             get: new GetWIDPAdminDefaultUseCase(widpAdminDefaultRepository),
             save: new SaveWIDPAdminDefaultCsvUseCase(widpAdminDefaultRepository),

src/data/DataQualityDefaultRepository.ts

@@ -0,0 +1,234 @@
+import { D2Api, GetOptions, Selector } from "../types/d2-api";
+import { CsvWriterDataSource } from "./CsvWriterCsvDataSource";
+import { downloadFile } from "./utils/download-file";
+import { CsvData } from "../data/CsvDataSource";
+import { ValidationResults } from "../domain/common/entities/ValidationResults";
+import { DataQualityRepository } from "../domain/data-quality/repositories/DataQualityRepository";
+import { Namespaces } from "./clients/storage/Namespaces";
+import { PersistedConfig } from "./entities/PersistedConfig";
+import { DataStoreStorageClient } from "./clients/storage/DataStoreStorageClient";
+import { StorageClient } from "./clients/storage/StorageClient";
+import { Instance } from "./entities/Instance";
+
+export async function promiseMap<T, S>(inputValues: T[], mapper: (value: T) => Promise<S>): Promise<S[]> {
+    const output: S[] = [];
+    for (const value of inputValues) {
+        const res = await mapper(value);
+        output.push(res);
+    }
+    return output;
+}
+export class DataQualityDefaultRepository implements DataQualityRepository {
+    private storageClient: StorageClient;
+
+    constructor(private api: D2Api) {
+        const instance = new Instance({ url: this.api.baseUrl });
+        this.storageClient = new DataStoreStorageClient("global", instance);
+    }
+
+    async reloadValidations(): Promise<ValidationResults[]> {
+        await this.storageClient.saveObject<PersistedConfig>(Namespaces.DATA_QUALITY_CONFIG, {});
+        return await this.getValidations();
+    }
+
+    async getValidations(): Promise<ValidationResults[]> {
+        const programIndicatorDate = this.getParsedDate(new Date());
+        const programIndicatorsvalidationsResult = await this.getValidatedProgramIndicators();
+        const indicatorDate = this.getParsedDate(new Date());
+        const indicatorsValidationsResult = await this.getValidatedIndicators();
+        const result: ValidationResults[] = [...indicatorsValidationsResult, ...programIndicatorsvalidationsResult];
+        const persistedData = await this.saveResult(result, indicatorDate, programIndicatorDate);
+        return persistedData;
+    }
+
+    async getValidatedIndicators(): Promise<ValidationResults[]> {
+        const startDate = await this.getIndicatorsLastUpdated();
+
+        const indicatorsResult = await this.api.models.indicators
+            .get({
+                fields: {
+                    id: true,
+                    numerator: true,
+                    denominator: true,
+                    name: true,
+                    user: true,
+                    lastUpdated: true,
+                },
+                filter: {
+                    lastUpdated: { gt: startDate },
+                },
+                paging: false,
+            })
+            .getData();
+
+        const d2api = this.api;
+        const results = await promiseMap(indicatorsResult.objects, async indicator => {
+            const numeratorResult =
+                indicator.numerator === ""
+                    ? undefined
+                    : await d2api.expressions.validate("indicator", indicator.numerator).getData();
+            const numerator = numeratorResult === undefined ? false : numeratorResult.message === "Valid";
+            const denominatorResult =
+                indicator.denominator === ""
+                    ? undefined
+                    : await d2api.expressions.validate("indicator", indicator.denominator).getData();
+            const denominator = denominatorResult === undefined ? false : denominatorResult.message === "Valid";
+
+            return {
+                metadataType: "Indicator",
+                id: indicator.id,
+                name: indicator.name,
+                numerator: indicator.numerator,
+                numeratorresult: numerator,
+                denominator: indicator.denominator,
+                denominatorresult: denominator,
+                user: indicator.user.id,
+                lastUpdated: indicator.lastUpdated,
+            };
+        });
+
+        return results;
+    }
+
+    async getValidatedProgramIndicators(): Promise<ValidationResults[]> {
+        const startDate = await this.getProgramIndicatorsLastUpdated();
+        const programIndicatorsResult = await this.api.models.programIndicators
+            .get({
+                fields: {
+                    id: true,
+                    filter: true,
+                    expression: true,
+                    name: true,
+                    user: true,
+                    lastUpdated: true,
+                },
+                filter: {
+                    lastUpdated: { gt: startDate },
+                },
+                paging: false,
+            })
+            .getData();
+
+        const d2api = this.api;
+        const results = await promiseMap(programIndicatorsResult.objects, async programIndicator => {
+            const expressionResult =
+                !("expression" in programIndicator) || programIndicator.expression === ""
+                    ? undefined
+                    : await d2api.expressions
+                          .validate("program-indicator-formula", programIndicator.expression)
+                          .getData();
+            const expression = expressionResult === undefined ? false : expressionResult.message === "Valid";
+            const filterResult =
+                !("filter" in programIndicator) || programIndicator.filter === ""
+                    ? undefined
+                    : await d2api.expressions.validate("program-indicator-filter", programIndicator.filter).getData();
+            const filter = filterResult === undefined ? false : filterResult.message === "Valid";
+            return {
+                metadataType: "ProgramIndicator",
+                id: programIndicator.id,
+                name: programIndicator.name,
+                expression: programIndicator.expression,
+                expressionresult: expression,
+                filter: programIndicator.filter,
+                filterresult: filter,
+                user: programIndicator.user.id,
+                lastUpdated: programIndicator.lastUpdated,
+            };
+        });
+        return results;
+    }
+
+    private getParsedDate(strDate: Date) {
+        return (
+            strDate.getFullYear() +
+            "-" +
+            ("0" + (new Date().getMonth() + 1)).slice(-2) +
+            "-" +
+            ("0" + new Date().getDate()).slice(-2)
+        );
+    }
+
+    private async getIndicatorsLastUpdated(): Promise<string> {
+        const { indicatorsLastUpdated = "1970-01-01" } = await this.getConfig();
+        return indicatorsLastUpdated;
+    }
+
+    private async getProgramIndicatorsLastUpdated(): Promise<string> {
+        const { programIndicatorsLastUpdated = "1970-01-01" } = await this.getConfig();
+        return programIndicatorsLastUpdated;
+    }
+
+    private async saveResult(
+        validationResults: ValidationResults[],
+        indicatorsLastUpdated: string,
+        programIndicatorsLastUpdated: string
+    ): Promise<ValidationResults[]> {
+        const config = await this.getConfig();
+        config.indicatorsLastUpdated = indicatorsLastUpdated;
+        config.programIndicatorsLastUpdated = programIndicatorsLastUpdated;
+
+        config.validationResults = config.validationResults
+            ? [...config.validationResults, ...validationResults]
+            : validationResults;
+
+        await this.storageClient.saveObject<PersistedConfig>(Namespaces.DATA_QUALITY_CONFIG, {
+            ...config,
+        });
+        return config.validationResults;
+    }
+
+    private async getConfig(): Promise<PersistedConfig> {
+        const config = await this.storageClient.getObject<PersistedConfig>(Namespaces.DATA_QUALITY_CONFIG);
+        return config ?? {};
+    }
+
+    async exportToCsv(): Promise<void> {
+        const metadataObjects = await (await this.getConfig()).validationResults;
+        const headers = csvFields.map(field => ({ id: field, text: field }));
+        if (metadataObjects === undefined) {
+            return;
+        } else {
+            const rows = metadataObjects.map(
+                (ValidationResults): MetadataRow => ({
+                    metadataType: ValidationResults.metadataType,
+                    id: ValidationResults.id,
+                    name: ValidationResults.name,
+                    expression: ValidationResults.expression,
+                    expressionresult: ValidationResults.expressionresult ? "valid" : "invalid",
+                    filter: ValidationResults.filter,
+                    filterresult: ValidationResults.filterresult ? "valid" : "invalid",
+                    numerator: ValidationResults.numerator,
+                    numeratorresult: ValidationResults.numeratorresult ? "valid" : "invalid",
+                    denominator: ValidationResults.denominator,
+                    denominatorresult: ValidationResults.denominatorresult ? "valid" : "invalid",
+                    user: ValidationResults.user,
+                })
+            );
+
+            const csvDataSource = new CsvWriterDataSource();
+            const csvData: CsvData<CsvField> = { headers, rows };
+            const csvContents = csvDataSource.toString(csvData);
+
+            await downloadFile(csvContents, "export.csv", "text/csv");
+        }
+    }
+}
+
+const csvFields = [
+    "metadataType",
+    "id",
+    "name",
+    "expression",
+    "expressionresult",
+    "filter",
+    "filterresult",
+    "numerator",
+    "numeratorresult",
+    "denominator",
+    "denominatorresult",
+    "user",
+] as const;
+
+type CsvField = typeof csvFields[number];
+
+type MetadataRow = Record<CsvField, string | undefined>;