diff --git a/README.md b/README.md index 0d0ba6f..0f1c9b4 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,16 @@ Expected format of `xlsx` file: ## Events +### Detect events assigned to organisation units outside their enrollment + +When enrollments are transferred to another org unit, the existing events keep their original org unit. While that's the expected default behaviour, sometimes we need to detect and fix these mismatches: + +```shell +$ yarn start:dev events detect-orgunits-outside-enrollment \ + --url "http://localhost:8080" --auth "USER:PASS" \ + --notify-email="SUBJECT,EMAIL1,EMAIL2,..." +``` + ### Move events from one orgunit to another Move events for program events (so no enrollments/TEIs move is supported): diff --git a/src/data/CategoryOptionD2Repository.ts b/src/data/CategoryOptionD2Repository.ts index ba16a8b..b7bb157 100644 --- a/src/data/CategoryOptionD2Repository.ts +++ b/src/data/CategoryOptionD2Repository.ts @@ -74,26 +74,28 @@ export class CategoryOptionD2Repository implements CategoryOptionRepository { { recordsSkipped: saveResponse.status === "ERROR" ? catOptionsToSave.map(co => co.id) : [], errorMessage, - created: saveResponse.response.stats.created, - ignored: saveResponse.response.stats.ignored, - updated: saveResponse.response.stats.updated, + ...saveResponse.response.stats, }, ]; }); return stats.reduce( - (acum, stat) => { + (acum, stat): Stats => { return { recordsSkipped: [...acum.recordsSkipped, ...stat.recordsSkipped], errorMessage: `${acum.errorMessage}${stat.errorMessage}`, created: acum.created + stat.created, ignored: acum.ignored + stat.ignored, updated: acum.updated + stat.updated, + deleted: acum.deleted + stat.deleted, + total: acum.total + stat.total, }; }, { recordsSkipped: [], errorMessage: "", + deleted: 0, + total: 0, created: 0, ignored: 0, updated: 0, diff --git a/src/data/D2Tracker.ts b/src/data/D2Tracker.ts index e33109a..cb0c192 100644 --- a/src/data/D2Tracker.ts +++ b/src/data/D2Tracker.ts @@ -5,11 +5,18 @@ import { Async } from "domain/entities/Async"; import { Stats } from "domain/entities/Stats"; import { D2Api } from "types/d2-api"; import log from "utils/log"; +import { TrackerPostRequest } from "@eyeseetea/d2-api/api/tracker"; +import { TrackedEntitiesGetResponse } from "@eyeseetea/d2-api/api/trackerTrackedEntities"; +import { TrackerEnrollmentsResponse } from "@eyeseetea/d2-api/api/trackerEnrollments"; +import { TrackerEventsResponse } from "@eyeseetea/d2-api/api/trackerEvents"; export class D2Tracker { constructor(private api: D2Api) {} - async postTracker(key: TrackerDataKey, objects: object[]): Async { + async postTracker( + key: Key, + objects: Array[number]> + ): Async { const total = objects.length; log.info(`Import data: ${key} - Total: ${total}`); let page = 1; @@ -29,17 +36,21 @@ export class D2Tracker { return result; } - private async postTrackerData(data: object, options: { payloadId: string }): Async { - const response: TrackerResponse = await this.api - .post("/tracker", { async: false }, data) + private async postTrackerData( + data: TrackerPostRequest, + options: { payloadId: string } + ): Async { + const response: TrackerResponse = await this.api.tracker + .post({ async: false }, data) .getData() - .catch(err => { - if (err?.response?.data) { - return err.response.data as TrackerResponse; + .then(res => ({ ...res, stats: { ...Stats.empty(), ...res.stats } })) + .catch((err): TrackerResponse => { + const data = err?.response?.data; + if (data) { + return data; } else { return { status: "ERROR", - typeReports: [], stats: Stats.empty(), }; } @@ -56,17 +67,18 @@ export class D2Tracker { } } - async getFromTracker( - apiPath: string, + async getFromTracker( + model: Key, options: { programIds: string[]; orgUnitIds: string[] | undefined; - fields?: string; trackedEntity?: string | undefined; } - ): Promise { - const output = []; - const { programIds, orgUnitIds, fields = "*", trackedEntity } = options; + ): Promise> { + type Output = Array; + + const output: Output = []; + const { programIds, orgUnitIds, trackedEntity } = options; for (const programId of programIds) { let page = 1; @@ -74,19 +86,28 @@ export class D2Tracker { while (dataRemaining) { const pageSize = 1000; - log.debug(`GET ${apiPath} (pageSize=${pageSize}, page=${page})`); - - const { instances } = await this.api - .get<{ instances: T[] }>(`/tracker/${apiPath}`, { - page, - pageSize: pageSize, - ouMode: orgUnitIds ? "SELECTED" : "ALL", - orgUnit: orgUnitIds?.join(";"), - fields: fields, - program: programId, - trackedEntity, - }) - .getData(); + log.debug(`GET ${model} (pageSize=${pageSize}, page=${page})`); + + const apiOptions = { + page, + pageSize: pageSize, + ouMode: orgUnitIds ? ("SELECTED" as const) : ("ALL" as const), + orgUnit: orgUnitIds?.join(";"), + fields: { $all: true as const }, + program: programId, + trackedEntity, + }; + + const { tracker } = this.api; + + const endpoint = { + trackedEntities: () => tracker.trackedEntities.get(apiOptions), + enrollments: () => tracker.enrollments.get(apiOptions), + events: () => tracker.events.get(apiOptions), + }; + + const res = await endpoint[model]().getData(); + const instances: Output = res.instances as Output; if (instances.length === 0) { dataRemaining = false; @@ -96,7 +117,7 @@ export class D2Tracker { } } } - log.info(`GET ${apiPath} -> Total: ${output.length}`); + log.info(`GET ${model} -> Total: ${output.length}`); return output; } @@ -104,8 +125,13 @@ export class D2Tracker { type TrackerResponse = { status: string; - typeReports: object[]; stats: Stats; }; type TrackerDataKey = "events" | "enrollments" | "trackedEntities"; + +type Mapping = { + trackedEntities: TrackedEntitiesGetResponse<{ $all: true }>["instances"]; + enrollments: TrackerEnrollmentsResponse<{ $all: true }>["instances"]; + events: TrackerEventsResponse<{ $all: true }>["instances"]; +}; diff --git a/src/data/ProgramEventsD2Repository.ts b/src/data/ProgramEventsD2Repository.ts index cb34862..1591bbd 100644 --- a/src/data/ProgramEventsD2Repository.ts +++ b/src/data/ProgramEventsD2Repository.ts @@ -1,91 +1,55 @@ import _ from "lodash"; -import { EventsGetResponse, PaginatedEventsGetResponse } from "@eyeseetea/d2-api/api/events"; import { Async } from "domain/entities/Async"; import { ProgramEvent } from "domain/entities/ProgramEvent"; import { GetOptions, ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository"; -import { D2Api, EventsPostRequest, EventsPostParams, Ref } from "types/d2-api"; +import { D2Api, Ref } from "types/d2-api"; import { cartesianProduct } from "utils/array"; import logger from "utils/log"; -import { getId, Id } from "domain/entities/Base"; +import { getId, Id, NamedRef } from "domain/entities/Base"; import { Result } from "domain/entities/Result"; -import { Timestamp } from "domain/entities/Date"; import { getInChunks } from "./dhis2-utils"; import { promiseMap } from "./dhis2-utils"; +import { TrackerPostParams, TrackerPostRequest } from "@eyeseetea/d2-api/api/tracker"; +import { TrackerEventsResponse } from "@eyeseetea/d2-api/api/trackerEvents"; const eventFields = { - created: true, + createdAt: true, event: true, status: true, orgUnit: true, orgUnitName: true, program: true, programStage: true, - eventDate: true, - dueDate: true, + occurredAt: true, + scheduledAt: true, lastUpdated: true, - trackedEntityInstance: true, + trackedEntity: true, dataValues: { dataElement: true, value: true, storedBy: true, providedElsewhere: true, - lastUpdated: true, + updatedAt: true, }, } as const; type Fields = typeof eventFields; -type Event = EventsGetResponse["events"][number]; +type Event = TrackerEventsResponse["instances"][number]; export class ProgramEventsD2Repository implements ProgramEventsRepository { constructor(private api: D2Api) {} async get(options: GetOptions): Async { + const d2EventsMapper = await D2EventsMapper.build(this.api); const d2Events = await this.getD2Events(options); - const { programs } = await this.api.metadata - .get({ - programs: { - fields: { - id: true, - name: true, - programStages: { id: true, name: true }, - }, - }, - }) - .getData(); - - const programsById = _.keyBy(programs, getId); - - const programStagesById = _(programs) - .flatMap(program => program.programStages) - .uniqBy(getId) - .keyBy(getId) - .value(); - - return d2Events.map(event => ({ - created: event.created, - id: event.event, - program: programsById[event.program] || { id: event.program, name: "" }, - programStage: programStagesById[event.programStage] || { id: event.programStage, name: "" }, - orgUnit: { id: event.orgUnit, name: event.orgUnitName }, - trackedEntityInstanceId: (event as D2Event).trackedEntityInstance, - status: event.status, - date: event.eventDate, - dueDate: event.dueDate, - dataValues: event.dataValues.map(dv => ({ - dataElementId: dv.dataElement, - value: dv.value, - storedBy: dv.storedBy, - providedElsewhere: dv.providedElsewhere, - lastUpdated: dv.lastUpdated, - })), - })); + return d2Events.map(d2Event => d2EventsMapper.getEventEntityFromD2Object(d2Event)); } async delete(events: Ref[]): Async { const d2Events = events.map(ev => ({ event: ev.id })) as EventToPost[]; - return importEvents(this.api, d2Events, { strategy: "DELETE" }); + return importEvents(this.api, d2Events, { importStrategy: "DELETE" }); } async save(events: ProgramEvent[]): Async { @@ -98,35 +62,33 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository { return this.getEvents(eventIds) .then(res => { const postEvents = eventIds.map((eventId): EventToPost => { - const existingD2Event = res.events.find(d2Event => d2Event.event === eventId); + const existingD2Event = res.instances.find(d2Event => d2Event.event === eventId); const event = eventsById[eventId]; if (!event) { throw Error("Cannot find event"); } return { - ...(existingD2Event || {}), + ...existingD2Event, event: event.id, program: event.program.id, programStage: event.programStage.id, orgUnit: event.orgUnit.id, status: event.status, - dueDate: event.dueDate, - eventDate: event.date, - dataValues: event.dataValues.map(dv => { - return { - dataElement: dv.dataElementId, - value: dv.value, - storedBy: dv.storedBy, - providedElsewhere: dv.providedElsewhere, - lastUpdated: dv.lastUpdated, - }; - }), + scheduledAt: event.dueDate, + occurredAt: event.date, + dataValues: event.dataValues.map(dv => ({ + dataElement: dv.dataElementId, + value: dv.value, + storedBy: dv.storedBy, + providedElsewhere: dv.providedElsewhere, + updatedAt: dv.lastUpdated, + })), }; }); return postEvents; }) .then(eventsToSave => { - return importEvents(this.api, eventsToSave, { strategy: "CREATE_AND_UPDATE" }); + return importEvents(this.api, eventsToSave, { importStrategy: "CREATE_AND_UPDATE" }); }) .then(responses => { return [responses]; @@ -134,12 +96,7 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository { .catch(() => { const message = `Error getting events: ${eventIds.join(",")}`; console.error(message); - return [ - { - type: "error", - message, - }, - ]; + return [{ type: "error", message }]; }); }); @@ -186,49 +143,102 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository { event: options.eventsIds?.join(";"), }; logger.debug(`Get API events: ${JSON.stringify(getEventsOptions)}`); - const { pager, events } = await this.api.events.get(getEventsOptions).getData(); + const res = await this.api.tracker.events.get(getEventsOptions).getData(); + const pageCount = Math.ceil((res.total || 0) / res.pageSize); - allEvents.push(...events); + allEvents.push(...res.instances); page++; - if (pager.page >= pager.pageCount) pendingPages = false; + if (res.page >= pageCount) pendingPages = false; } } return allEvents; } - private getEvents(eventIds: Id[]): Async> { - return this.api.events + private getEvents(eventIds: Id[]) { + return this.api.tracker.events .get({ event: eventIds.join(";"), fields: eventFields, totalPages: true, - pageSize: 1e6, + pageSize: eventIds.length, }) .getData(); } } -interface D2Event { - event: string; - status: Event["status"]; - orgUnit: Id; - orgUnitName: string; - program: Id; - dataValues: Array<{ dataElement: Id; value: string; storedBy: string }>; - eventDate: Timestamp; - dueDate: Timestamp; - trackedEntityInstance?: Id; +export class D2EventsMapper { + constructor( + private programsById: Record, + private programStagesById: Record + ) {} + + static async build(api: D2Api) { + const { programs } = await api.metadata + .get({ + programs: { + fields: { + id: true, + name: true, + programStages: { id: true, name: true }, + }, + }, + }) + .getData(); + + const programsById = _.keyBy(programs, getId); + + const programStagesById = _(programs) + .flatMap(program => program.programStages) + .uniqBy(getId) + .keyBy(getId) + .value(); + + return new D2EventsMapper(programsById, programStagesById); + } + + getEventEntityFromD2Object(event: Event): ProgramEvent { + return { + id: event.event, + created: event.createdAt, + program: this.programsById[event.program] || { id: event.program, name: "" }, + programStage: this.programStagesById[event.programStage] || { id: event.programStage, name: "" }, + orgUnit: { id: event.orgUnit, name: event.orgUnitName }, + trackedEntityInstanceId: event.trackedEntity, + status: event.status, + date: event.occurredAt, + dueDate: event.scheduledAt, + dataValues: event.dataValues.map(dv => ({ + dataElementId: dv.dataElement, + value: dv.value, + storedBy: dv.storedBy, + providedElsewhere: dv.providedElsewhere, + lastUpdated: dv.updatedAt, + })), + }; + } } -type EventToPost = EventsPostRequest["events"][number] & { event: Id; dueDate: Timestamp }; +type EventToPost = NonNullable[number]; -async function importEvents(api: D2Api, events: EventToPost[], params?: EventsPostParams): Async { +async function importEvents(api: D2Api, events: EventToPost[], params?: TrackerPostParams): Async { if (_.isEmpty(events)) return { type: "success", message: "No events to post" }; const resList = await promiseMap(_.chunk(events, 100), async eventsGroup => { - const res = await api.events.post(params || {}, { events: eventsGroup }).getData(); - if (res.status === "SUCCESS") { + const res = await api.tracker + .post( + { + async: false, + skipPatternValidation: true, + skipSideEffects: true, + skipRuleEngine: true, + importMode: "COMMIT", + ...params, + }, + { events: eventsGroup } + ) + .getData(); + if (res.status === "OK") { const message = JSON.stringify( _.pick(res, ["status", "imported", "updated", "deleted", "ignored"]) ); diff --git a/src/data/ProgramsD2Repository.ts b/src/data/ProgramsD2Repository.ts index 45e4749..3d99fd4 100644 --- a/src/data/ProgramsD2Repository.ts +++ b/src/data/ProgramsD2Repository.ts @@ -8,6 +8,10 @@ import log from "utils/log"; import { promiseMap, runMetadata } from "./dhis2-utils"; import { D2ProgramRules } from "./d2-program-rules/D2ProgramRules"; import { D2Tracker } from "./D2Tracker"; +import { Program, ProgramType } from "domain/entities/Program"; +import { D2TrackerEnrollmentToPost } from "@eyeseetea/d2-api/api/trackerEnrollments"; +import { D2TrackerEventToPost } from "@eyeseetea/d2-api/api/trackerEvents"; +import { D2TrackedEntityInstanceToPost } from "@eyeseetea/d2-api/api/trackerTrackedEntities"; type MetadataRes = { date: string } & { [k: string]: Array<{ id: string }> }; @@ -18,17 +22,33 @@ export class ProgramsD2Repository implements ProgramsRepository { this.d2Tracker = new D2Tracker(this.api); } + async get(options: { programTypes?: ProgramType[] }): Async { + const { programs } = await this.api.metadata + .get({ + programs: { + fields: { + id: true, + name: true, + programType: true, + }, + filter: { + ...(options.programTypes ? { programType: { in: options.programTypes } } : {}), + }, + }, + }) + .getData(); + + return programs; + } + async export(options: { ids: Id[]; orgUnitIds: Id[] | undefined }): Async { const { ids: programIds, orgUnitIds } = options; const metadata = await this.getMetadata(programIds); const getOptions = { programIds, orgUnitIds }; - const events = await this.d2Tracker.getFromTracker("events", getOptions); - const enrollments = await this.d2Tracker.getFromTracker("enrollments", getOptions); - const trackedEntities = await this.d2Tracker.getFromTracker( - "trackedEntities", - getOptions - ); + const events = await this.d2Tracker.getFromTracker("events", getOptions); + const enrollments = await this.d2Tracker.getFromTracker("enrollments", getOptions); + const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", getOptions); /* Remove redundant enrollments info from TEIs */ const trackedEntitiesWithoutEnrollments = trackedEntities.map(trackedEntity => ({ @@ -39,7 +59,7 @@ export class ProgramsD2Repository implements ProgramsRepository { return { metadata, data: { - events, + events: events, enrollments: enrollments, trackedEntities: trackedEntitiesWithoutEnrollments, }, @@ -78,7 +98,7 @@ export class ProgramsD2Repository implements ProgramsRepository { // DHIS2 exports enrollments without attributes, but requires it on import, add from TEI const enrollmentsWithAttributes = enrollments.map(enrollment => ({ ...enrollment, - attributes: teisById[enrollment.trackedEntity]?.attributes || [], + attributes: (enrollment.trackedEntity && teisById[enrollment.trackedEntity]?.attributes) || [], })); log.info(`Import data`); @@ -99,16 +119,11 @@ interface D2ProgramExport { } type D2ProgramData = { - events: object[]; - enrollments: D2Enrollment[]; - trackedEntities: D2TrackedEntity[]; + events: D2TrackerEventToPost[]; + enrollments: D2TrackerEnrollmentToPost[]; + trackedEntities: D2TrackedEntityInstanceToPost[]; }; -interface D2Enrollment { - enrollment: string; - trackedEntity: string; -} - export interface D2TrackedEntity { trackedEntity: Id; orgUnit: Id; diff --git a/src/data/TrackedEntityD2Repository.ts b/src/data/TrackedEntityD2Repository.ts index ee83d77..8d16039 100644 --- a/src/data/TrackedEntityD2Repository.ts +++ b/src/data/TrackedEntityD2Repository.ts @@ -8,9 +8,9 @@ import { TrackedEntityFilterParams, TrackedEntityRepository, } from "domain/repositories/TrackedEntityRepository"; -import { TrackedEntity } from "domain/entities/TrackedEntity"; -import { D2TrackedEntity } from "./ProgramsD2Repository"; +import { Enrollment, TrackedEntity } from "domain/entities/TrackedEntity"; import { D2Tracker } from "./D2Tracker"; +import { D2EventsMapper } from "./ProgramEventsD2Repository"; export class TrackedEntityD2Repository implements TrackedEntityRepository { private d2Tracker: D2Tracker; @@ -20,15 +20,26 @@ export class TrackedEntityD2Repository implements TrackedEntityRepository { } async getAll(params: TrackedEntityFilterParams): Async { - const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", { + const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", { orgUnitIds: undefined, programIds: [params.programId], }); + const d2EventsMapper = await D2EventsMapper.build(this.api); + return trackedEntities.map(tei => { return { id: tei.trackedEntity, orgUnit: tei.orgUnit, + enrollments: tei.enrollments.map( + (enrollment): Enrollment => ({ + id: enrollment.enrollment, + orgUnit: { id: enrollment.orgUnit, name: enrollment.orgUnitName }, + events: enrollment.events.map(event => + d2EventsMapper.getEventEntityFromD2Object(event) + ), + }) + ), trackedEntityType: tei.trackedEntityType, attributes: tei.attributes.map(attribute => { return { @@ -55,7 +66,7 @@ export class TrackedEntityD2Repository implements TrackedEntityRepository { .value(); const stats = await getInChunks(teisToFetch, async teiIds => { - const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", { + const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", { orgUnitIds: undefined, programIds: programsIds, trackedEntity: teiIds.join(";"), @@ -69,6 +80,7 @@ export class TrackedEntityD2Repository implements TrackedEntityRepository { }); return { + ...tei, trackedEntity: currentProgram.id, orgUnit: tei.orgUnit, trackedEntityType: tei.trackedEntityType, @@ -80,13 +92,14 @@ export class TrackedEntityD2Repository implements TrackedEntityRepository { return _(response) .map(item => { - const isError = item.status === "ERROR"; return new Stats({ created: item.stats.created, updated: item.stats.updated, ignored: item.stats.ignored, - errorMessage: isError ? item.status : "", - recordsSkipped: isError ? teisToSave.map(tei => tei.trackedEntity) : [], + deleted: item.stats.deleted, + total: item.stats.total, + recordsSkipped: [], + errorMessage: "", }); }) .value(); diff --git a/src/data/UserD2Repository.ts b/src/data/UserD2Repository.ts index 76928e9..6170af7 100644 --- a/src/data/UserD2Repository.ts +++ b/src/data/UserD2Repository.ts @@ -116,7 +116,7 @@ export class UserD2Repository implements UserRepository { return { usersToSave, response }; }); }) - .then(result => { + .then((result): Stats[] => { const response = result.response.data; const errorMessage = response.typeReports .flatMap(x => x.objectReports) @@ -129,22 +129,19 @@ export class UserD2Repository implements UserRepository { recordsSkipped: response.status === "ERROR" ? result.usersToSave.map(user => user.id) : [], errorMessage, - created: response.stats.created, - ignored: response.stats.ignored, - updated: response.stats.updated, + ...response.stats, }, ]; }) - .catch(err => { + .catch((err): Stats[] => { const errorMessage = `Error getting users ${userIds.join(",")}`; console.error(errorMessage, err); return [ { + ...Stats.empty(), recordsSkipped: userIds, errorMessage, - created: 0, ignored: userIds.length, - updated: 0, }, ]; }); diff --git a/src/data/user-monitoring/permission-fixer/PermissionFixerUserGroupD2Repository.ts b/src/data/user-monitoring/permission-fixer/PermissionFixerUserGroupD2Repository.ts index 2e4adea..230548d 100644 --- a/src/data/user-monitoring/permission-fixer/PermissionFixerUserGroupD2Repository.ts +++ b/src/data/user-monitoring/permission-fixer/PermissionFixerUserGroupD2Repository.ts @@ -1,3 +1,4 @@ +import _ from "lodash"; import { D2Api } from "@eyeseetea/d2-api/2.36"; import log from "utils/log"; import { PermissionFixerUserGroupExtended } from "domain/entities/user-monitoring/permission-fixer/PermissionFixerUserGroupExtended"; @@ -26,15 +27,15 @@ export class PermissionFixerUserGroupD2Repository implements PermissionFixerUser async save(userGroup: PermissionFixerUserGroupExtended): Async { try { const response = await this.api.models.userGroups.put(userGroup).getData(); - if (response.status == "OK") { + if (_(response.errorReports).isEmpty()) { log.info("Users added to minimal group"); } else { log.error("Error adding users to minimal group"); } - log.info(JSON.stringify(response.response)); + log.info(JSON.stringify(response)); - return response.status; + return "SUCCESS"; } catch (error) { console.debug(error); return "ERROR"; diff --git a/src/domain/entities/Program.ts b/src/domain/entities/Program.ts new file mode 100644 index 0000000..7eda7cd --- /dev/null +++ b/src/domain/entities/Program.ts @@ -0,0 +1,9 @@ +import { Id } from "./Base"; + +export type ProgramType = "WITH_REGISTRATION" | "WITHOUT_REGISTRATION"; + +export interface Program { + id: Id; + name: string; + programType: ProgramType; +} diff --git a/src/domain/entities/ProgramEvent.ts b/src/domain/entities/ProgramEvent.ts index 2108233..8e59abb 100644 --- a/src/domain/entities/ProgramEvent.ts +++ b/src/domain/entities/ProgramEvent.ts @@ -28,7 +28,7 @@ export interface ProgramEventToSave { dueDate: Timestamp; } -type EventStatus = "ACTIVE" | "COMPLETED" | "VISITED" | "SCHEDULED" | "OVERDUE" | "SKIPPED"; +type EventStatus = "ACTIVE" | "COMPLETED" | "VISITED" | "SCHEDULE" | "OVERDUE" | "SKIPPED"; export const orgUnitModes = ["SELECTED", "CHILDREN", "DESCENDANTS"] as const; diff --git a/src/domain/entities/Stats.ts b/src/domain/entities/Stats.ts index a5ecfc4..3e6cab3 100644 --- a/src/domain/entities/Stats.ts +++ b/src/domain/entities/Stats.ts @@ -2,25 +2,31 @@ import { Id } from "./Base"; type StatsAttrs = { recordsSkipped: Id[]; + errorMessage: string; created: number; ignored: number; updated: number; - errorMessage: string; + deleted: number; + total: number; }; export class Stats { public readonly recordsSkipped: Id[]; + public readonly errorMessage: string; public readonly created: number; public readonly ignored: number; public readonly updated: number; - public readonly errorMessage: string; + public readonly deleted: number; + public readonly total: number; constructor(attrs: StatsAttrs) { this.recordsSkipped = attrs.recordsSkipped; this.created = attrs.created; this.ignored = attrs.ignored; this.updated = attrs.updated; + this.deleted = attrs.deleted; this.errorMessage = attrs.errorMessage; + this.total = attrs.total; } static combine(stats: Stats[]): Stats { @@ -31,6 +37,8 @@ export class Stats { created: acum.created + stat.created, ignored: acum.ignored + stat.ignored, updated: acum.updated + stat.updated, + deleted: acum.deleted + stat.deleted, + total: acum.total + stat.total, }; }, Stats.empty()); } @@ -42,6 +50,8 @@ export class Stats { created: 0, ignored: 0, updated: 0, + deleted: 0, + total: 0, }; } } diff --git a/src/domain/entities/TrackedEntity.ts b/src/domain/entities/TrackedEntity.ts index 5b51989..e97d8fe 100644 --- a/src/domain/entities/TrackedEntity.ts +++ b/src/domain/entities/TrackedEntity.ts @@ -1,4 +1,5 @@ -import { Id, Ref } from "./Base"; +import { Id, NamedRef, Ref } from "./Base"; +import { ProgramEvent } from "./ProgramEvent"; export interface TrackedEntity extends Ref { id: Id; @@ -6,8 +7,15 @@ export interface TrackedEntity extends Ref { orgUnit: Id; trackedEntityType: Id; programId: Id; + enrollments: Enrollment[]; } +export type Enrollment = { + id: Id; + orgUnit: NamedRef; + events: ProgramEvent[]; +}; + export type AttributeValue = { attributeId: Id; value: string; diff --git a/src/domain/repositories/ProgramsRepository.ts b/src/domain/repositories/ProgramsRepository.ts index 3b05c68..8ad280e 100644 --- a/src/domain/repositories/ProgramsRepository.ts +++ b/src/domain/repositories/ProgramsRepository.ts @@ -1,9 +1,11 @@ import { Async } from "domain/entities/Async"; import { Id } from "domain/entities/Base"; import { Timestamp } from "domain/entities/Date"; +import { Program, ProgramType } from "domain/entities/Program"; import { ProgramExport } from "domain/entities/ProgramExport"; export interface ProgramsRepository { + get(options: { programTypes?: ProgramType[] }): Async; export(options: { ids: Id[] }): Async; import(programExport: ProgramExport): Async; runRules(options: RunRulesOptions): Async; diff --git a/src/domain/repositories/TrackedEntityRepository.ts b/src/domain/repositories/TrackedEntityRepository.ts index 06f25d3..e3838ec 100644 --- a/src/domain/repositories/TrackedEntityRepository.ts +++ b/src/domain/repositories/TrackedEntityRepository.ts @@ -10,6 +10,4 @@ export interface TrackedEntityRepository { export type TrackedEntityFilterParams = { programId: Id; - fromAttributeId: Id; - toAttributeId: Id; }; diff --git a/src/domain/usecases/MoveProgramAttributeUseCase.ts b/src/domain/usecases/MoveProgramAttributeUseCase.ts index 3bf56d8..66a4033 100644 --- a/src/domain/usecases/MoveProgramAttributeUseCase.ts +++ b/src/domain/usecases/MoveProgramAttributeUseCase.ts @@ -7,11 +7,16 @@ import { TrackedEntityFilterParams, TrackedEntityRepository, } from "domain/repositories/TrackedEntityRepository"; +import { Id } from "domain/entities/Base"; +export type MoveProgramAttributeUseCaseOptions = TrackedEntityFilterParams & { + fromAttributeId: Id; + toAttributeId: Id; +}; export class MoveProgramAttributeUseCase { constructor(private trackedEntityRepository: TrackedEntityRepository) {} - async execute(options: TrackedEntityFilterParams): Async { + async execute(options: MoveProgramAttributeUseCaseOptions): Async { const teis = await this.trackedEntityRepository.getAll(options); const teisWithCopyValues = _(teis) diff --git a/src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts b/src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts new file mode 100644 index 0000000..2981382 --- /dev/null +++ b/src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts @@ -0,0 +1,150 @@ +import _ from "lodash"; +import { promiseMap } from "data/dhis2-utils"; +import { NamedRef } from "domain/entities/Base"; +import logger from "utils/log"; +import { Maybe } from "utils/ts-utils"; +import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; +import { NotificationsRepository } from "domain/repositories/NotificationsRepository"; +import { ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository"; +import { TrackedEntityRepository } from "domain/repositories/TrackedEntityRepository"; +import { Enrollment, TrackedEntity } from "domain/entities/TrackedEntity"; +import { ProgramEvent, ProgramEventToSave } from "domain/entities/ProgramEvent"; + +export class DetectExternalOrgUnitUseCase { + constructor( + private programRepository: ProgramsRepository, + private trackedEntityRepository: TrackedEntityRepository, + private eventsRepository: ProgramEventsRepository, + private notificationRepository: NotificationsRepository + ) {} + + async execute(options: { + post: boolean; + notification: Maybe<{ subject: string; recipients: string[] }>; + }) { + const programs = await this.getPrograms(); + + const report = joinReports( + await promiseMap(programs, async program => { + return this.fixEventsInProgram({ program: program, post: options.post }); + }) + ); + + if (report.events > 0 && options.notification) { + const status = options.post ? "fixed" : "detected"; + + const body = [ + `${report.events} events outside its enrollment organisation unit [${status}]`, + "", + report.contents, + ]; + + await this.notificationRepository.send({ + recipients: options.notification.recipients, + subject: options.notification.subject, + body: { type: "text", contents: body.join("\n") }, + attachments: [], + }); + } + } + + private async getPrograms() { + logger.info(`Get tracker programs`); + const programs = await this.programRepository.get({ programTypes: ["WITH_REGISTRATION"] }); + logger.info(`Total tracker programs: ${programs.length}`); + return programs; + } + + async fixEventsInProgram(options: { program: NamedRef; post: boolean }): Promise { + const trackedEntities = await this.trackedEntityRepository.getAll({ programId: options.program.id }); + const mismatchRecords = this.getMismatchRecords(trackedEntities); + const report = this.buildReport(mismatchRecords); + logger.info(`Events outside its enrollment orgUnit: ${mismatchRecords.length}`); + logger.info(report); + + if (_(mismatchRecords).isEmpty()) { + logger.debug(`No events outside its enrollment orgUnit`); + } else if (!options.post) { + logger.info(`Add --post to update events (${mismatchRecords.length})`); + } else { + await this.fixMismatchEvents(mismatchRecords); + } + + return { contents: report, events: mismatchRecords.length }; + } + + private async fixMismatchEvents(mismatchRecords: MismatchRecord[]) { + const events = mismatchRecords.map(obj => obj.event); + const mismatchRecordsByEventId = _.keyBy(mismatchRecords, obj => obj.event.id); + + const fixedEvents = events.map((event): ProgramEventToSave => { + const obj = mismatchRecordsByEventId[event.id]; + if (!obj) throw new Error(`Event not found: ${event.id}`); + return { ...event, orgUnit: obj.enrollment.orgUnit }; + }); + + await this.saveEvents(fixedEvents); + } + + private async saveEvents(events: ProgramEventToSave[]) { + logger.info(`Post events: ${events.length}`); + const response = await this.eventsRepository.save(events); + logger.info(`Post result: ${JSON.stringify(response)}`); + } + + private buildReport(mismatchRecords: MismatchRecord[]): string { + return mismatchRecords + .map(obj => { + const { trackedEntity: tei, enrollment: enrollment, event } = obj; + + const msg = [ + `trackedEntity: id=${tei.id} orgUnit="${enrollment.orgUnit.name}" [${enrollment.orgUnit.id}]`, + `event: id=${event.id} orgUnit="${event.orgUnit.name}" [${event.orgUnit.id}]`, + ]; + + return msg.join(" - "); + }) + .join("\n"); + } + + private getMismatchRecords(trackedEntities: TrackedEntity[]): MismatchRecord[] { + return _(trackedEntities) + .flatMap(trackedEntity => { + return _(trackedEntity.enrollments) + .flatMap(enrollment => { + return enrollment.events.map(event => { + if (event.orgUnit !== enrollment.orgUnit) { + return { + trackedEntity: trackedEntity, + enrollment: enrollment, + event: event, + }; + } + }); + }) + .compact() + .value(); + }) + .value(); + } +} + +type MismatchRecord = { + trackedEntity: TrackedEntity; + enrollment: Enrollment; + event: ProgramEvent; +}; + +type Report = { contents: string; events: number }; + +function joinReports(reports: Report[]): Report { + return { + contents: _(reports) + .map(report => report.contents) + .compact() + .join("\n"), + events: _(reports) + .map(report => report.events) + .sum(), + }; +} diff --git a/src/scripts/commands/events.ts b/src/scripts/commands/events.ts index 15e5ef0..9352b56 100644 --- a/src/scripts/commands/events.ts +++ b/src/scripts/commands/events.ts @@ -1,12 +1,22 @@ import _ from "lodash"; import { command, string, subcommands, option, optional, flag } from "cmd-ts"; -import { getApiUrlOption, getD2Api, StringsSeparatedByCommas } from "scripts/common"; +import { + getApiUrlOption, + getApiUrlOptions, + getD2Api, + getD2ApiFromArgs, + StringsSeparatedByCommas, +} from "scripts/common"; import { ProgramEventsD2Repository } from "data/ProgramEventsD2Repository"; import { MoveEventsToOrgUnitUseCase } from "domain/usecases/MoveEventsToOrgUnitUseCase"; import logger from "utils/log"; import { UpdateEventDataValueUseCase } from "domain/usecases/UpdateEventDataValueUseCase"; import { EventExportSpreadsheetRepository } from "data/EventExportSpreadsheetRepository"; +import { DetectExternalOrgUnitUseCase } from "domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase"; +import { ProgramsD2Repository } from "data/ProgramsD2Repository"; +import { NotificationsEmailRepository } from "data/NotificationsEmailRepository"; +import { TrackedEntityD2Repository } from "data/TrackedEntityD2Repository"; export function getCommand() { return subcommands({ @@ -14,10 +24,50 @@ export function getCommand() { cmds: { "move-to-org-unit": moveOrgUnitCmd, "update-events": updateEventsDataValues, + "detect-orgunits-outside-enrollment": detectEventsOutsideOrgUnitEnrollmentCmd, }, }); } +const detectEventsOutsideOrgUnitEnrollmentCmd = command({ + name: "detect-external-orgunits", + description: "Detect events assigned to organisation units outside their enrollment", + args: { + ...getApiUrlOptions(), + post: flag({ + long: "post", + description: "Fix events", + defaultValue: () => false, + }), + notifyEmail: option({ + type: optional(StringsSeparatedByCommas), + long: "notify-email", + description: "SUBJECT,EMAIL1,EMAIL2,...", + }), + }, + handler: async args => { + const api = getD2ApiFromArgs(args); + const programsRepository = new ProgramsD2Repository(api); + const notificationRepository = new NotificationsEmailRepository(); + const eventsRepository = new ProgramEventsD2Repository(api); + const trackedEntitiesRepository = new TrackedEntityD2Repository(api); + const { notifyEmail } = args; + const [subject, ...recipients] = notifyEmail || []; + const notification = + subject && recipients.length > 0 ? { subject: subject, recipients: recipients } : undefined; + + return new DetectExternalOrgUnitUseCase( + programsRepository, + trackedEntitiesRepository, + eventsRepository, + notificationRepository + ).execute({ + ...args, + notification: notification, + }); + }, +}); + const moveOrgUnitCmd = command({ name: "move-to-org-unit", description: "Move events to another organisation unit for event programs",