Skip to content

Commit

Permalink
Merge pull request #641 from EyeSeeTea/development
Browse files Browse the repository at this point in the history 
Release 2.2.1
  • Loading branch information
@adrianq
adrianq authored Sep 24, 2020
2 parents a2024b4 + dd7edb8 commit 3228af9
Show file tree
Hide file tree
Showing 4 changed files with 189 additions and 96 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "metadata-synchronization",
"description": "Advanced metadata & data synchronization utility",
"version": "2.2.0",
"version": "2.2.1",
"license": "GPL-3.0",
"author": "EyeSeeTea team",
"homepage": ".",
Expand Down
87 changes: 76 additions & 11 deletions src/data/events/EventsD2ApiRepository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
} from "../../domain/synchronization/entities/SynchronizationResult";
import { cleanObjectDefault, cleanOrgUnitPaths } from "../../domain/synchronization/utils";
import { DataImportParams } from "../../types/d2";
import { D2Api } from "../../types/d2-api";
import { D2Api, Pager } from "../../types/d2-api";

export class EventsD2ApiRepository implements EventsRepository {
private api: D2Api;
Expand All @@ -24,31 +24,91 @@ export class EventsD2ApiRepository implements EventsRepository {
programs: string[] = [],
defaults: string[] = []
): Promise<ProgramEvent[]> {
const { period, orgUnitPaths = [], events = [], allEvents } = params;
const [startDate, endDate] = buildPeriodFromParams(params);
if (params.allEvents) return this.getAllEvents(params, programs, defaults);
else return this.getSpecificEvents(params, programs, defaults);
}

/**
* Design choices and heads-up:
* - The events endpoint does not support multiple values for a given filter
* meaning you cannot query for multiple programs or multiple orgUnits in
* the same API call. Instead you need to query one by one
* - Querying one by one is not performant, instead we query for all events
* available in the instance and manually filter them in this method
* - For big databases querying for all events available in a given instance
* with paging=false makes the instance to eventually go offline
* - Instead of disabling paging we traverse all the events by paginating all
* the available pages so that we can filter them afterwards
*/
private async getAllEvents(
params: DataSynchronizationParams,
programs: string[] = [],
defaults: string[] = []
): Promise<ProgramEvent[]> {
if (programs.length === 0) return [];

const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
const { period, orgUnitPaths = [] } = params;
const [startDate, endDate] = buildPeriodFromParams(params);

const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
const result = [];

for (const program of programs) {
const { events: response } = (await this.api
.get("/events", {
paging: false,
const fetchApi = async (program: string, page: number) => {
return this.api
.get<EventExportResult>("/events", {
pageSize: 250,
totalPages: true,
page,
program,
startDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined,
endDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined,
})
.getData()) as { events: (ProgramEvent & { event: string })[] };
.getData();
};

for (const program of programs) {
const { events, pager } = await fetchApi(program, 1);
result.push(...events);

result.push(...response);
for (let page = 2; page <= pager.pageCount; page += 1) {
const { events } = await fetchApi(program, page);
result.push(...events);
}
}

return _(result)
.filter(({ orgUnit }) => orgUnits.includes(orgUnit))
.map(object => ({ ...object, id: object.event }))
.map(object => cleanObjectDefault(object, defaults))
.value();
}

private async getSpecificEvents(
params: DataSynchronizationParams,
programs: string[] = [],
defaults: string[] = []
): Promise<ProgramEvent[]> {
const { orgUnitPaths = [], events: filter = [] } = params;
if (programs.length === 0 || filter.length === 0) return [];

const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
const result = [];

for (const program of programs) {
for (const ids of _.chunk(filter, 300)) {
const { events } = await this.api
.get<EventExportResult>("/events", {
paging: false,
program,
event: ids.join(";"),
})
.getData();
result.push(...events);
}
}

return _(result)
.filter(({ orgUnit }) => orgUnits.includes(orgUnit))
.filter(({ event }) => (allEvents ? true : events.includes(event)))
.map(object => ({ ...object, id: object.event }))
.map(object => cleanObjectDefault(object, defaults))
.value();
Expand Down Expand Up @@ -125,3 +185,8 @@ interface EventsPostResponse {
}[];
};
}

interface EventExportResult {
events: Array<ProgramEvent & { event: string }>;
pager: Pager;
}
2 changes: 2 additions & 0 deletions src/data/metadata/__tests__/integration/sync-events.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ describe("Sync metadata", () => {
remote.get("/dataValueSets", async () => ({ dataValues: [] }));

local.get("/events", async () => ({
pager: { page: 1, pageCount: 1, pageSize: 1, total: 1 },
events: [
{
storedBy: "widp.admin",
Expand Down Expand Up @@ -125,6 +126,7 @@ describe("Sync metadata", () => {
}));

remote.get("/events", async () => ({
pager: { page: 1, pageCount: 1, pageSize: 1, total: 1 },
events: [
{
storedBy: "widp.admin",
Expand Down
194 changes: 110 additions & 84 deletions src/presentation/react/components/sync-wizard/data/EventsSelectionStep.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { Typography } from "@material-ui/core";
import { ObjectsTable, ObjectsTableDetailField, TableColumn, TableState } from "d2-ui-components";
import _ from "lodash";
import React, { useEffect, useState } from "react";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import { ProgramEvent } from "../../../../../domain/events/entities/ProgramEvent";
import { DataElement, Program } from "../../../../../domain/metadata/entities/MetadataEntities";
import i18n from "../../../../../locales";
import SyncRule from "../../../../../models/syncRule";
import { useAppContext } from "../../../contexts/AppContext";
import Dropdown from "../../dropdown/Dropdown";
import { Toggle } from "../../toggle/Toggle";
Expand All @@ -20,30 +21,127 @@ type CustomProgram = Program & {

export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardStepProps) {
const { compositionRoot } = useAppContext();

const [memoizedSyncRule] = useState<SyncRule>(syncRule);
const [objects, setObjects] = useState<ProgramEvent[] | undefined>();
const [programs, setPrograms] = useState<CustomProgram[]>([]);
const [programFilter, changeProgramFilter] = useState<string>("");
const [error, setError] = useState();
const [error, setError] = useState<unknown>();

useEffect(() => {
const sync = compositionRoot.sync.events(memoizedSyncRule.toBuilder());
sync.extractMetadata<CustomProgram>().then(({ programs = [] }) => setPrograms(programs));
}, [memoizedSyncRule, compositionRoot]);

useEffect(() => {
if (programs.length === 0) return;
compositionRoot.events
.list(
{
...syncRule.dataParams,
...memoizedSyncRule.dataParams,
allEvents: true,
},
programs.map(({ id }) => id)
)
.then(setObjects)
.catch(setError);
}, [compositionRoot, syncRule, programs]);
}, [compositionRoot, memoizedSyncRule, programs]);

useEffect(() => {
const sync = compositionRoot.sync.events(syncRule.toBuilder());
sync.extractMetadata<CustomProgram>().then(({ programs = [] }) => setPrograms(programs));
}, [syncRule, compositionRoot]);
const handleTableChange = useCallback(
(tableState: TableState<ProgramEvent>) => {
const { selection } = tableState;
onChange(syncRule.updateDataSyncEvents(selection.map(({ id }) => id)));
},
[onChange, syncRule]
);

const updateSyncAll = useCallback(
(value: boolean) => {
onChange(syncRule.updateDataSyncAllEvents(value).updateDataSyncEvents(undefined));
},
[onChange, syncRule]
);

const addToSelection = useCallback(
(ids: string[]) => {
const oldSelection = _.difference(syncRule.dataSyncEvents, ids);
const newSelection = _.difference(ids, syncRule.dataSyncEvents);

onChange(syncRule.updateDataSyncEvents([...oldSelection, ...newSelection]));
},
[onChange, syncRule]
);

const columns: TableColumn<ProgramEvent>[] = useMemo(
() => [
{ name: "id" as const, text: i18n.t("UID"), sortable: true },
{
name: "program" as const,
text: i18n.t("Program"),
sortable: true,
getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
},
{ name: "orgUnitName" as const, text: i18n.t("Organisation unit"), sortable: true },
{ name: "eventDate" as const, text: i18n.t("Event date"), sortable: true },
{
name: "lastUpdated" as const,
text: i18n.t("Last updated"),
sortable: true,
hidden: true,
},
{ name: "status" as const, text: i18n.t("Status"), sortable: true },
{ name: "storedBy" as const, text: i18n.t("Stored by"), sortable: true },
],
[programs]
);

const details: ObjectsTableDetailField<ProgramEvent>[] = useMemo(
() => [
{ name: "id" as const, text: i18n.t("UID") },
{
name: "program" as const,
text: i18n.t("Program"),
getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
},
{ name: "orgUnitName" as const, text: i18n.t("Organisation unit") },
{ name: "created" as const, text: i18n.t("Created") },
{ name: "lastUpdated" as const, text: i18n.t("Last updated") },
{ name: "eventDate" as const, text: i18n.t("Event date") },
{ name: "dueDate" as const, text: i18n.t("Due date") },
{ name: "status" as const, text: i18n.t("Status") },
{ name: "storedBy" as const, text: i18n.t("Stored by") },
],
[programs]
);

const actions = useMemo(
() => [
{
name: "select",
text: i18n.t("Select"),
primary: true,
multiple: true,
onClick: addToSelection,
isActive: () => false,
},
],
[addToSelection]
);

const buildAdditionalColumns = () => {
const filterComponents = useMemo(
() => (
<Dropdown
key={"program-filter"}
items={programs}
onValueChange={changeProgramFilter}
value={programFilter}
label={i18n.t("Program")}
/>
),
[programFilter, programs]
);

const additionalColumns = useMemo(() => {
const program = _.find(programs, { id: programFilter });
const dataElements = _(program?.programStages ?? [])
.map(({ programStageDataElements }) =>
Expand All @@ -61,82 +159,8 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt
return _.find(row.dataValues, { dataElement: id })?.value ?? "-";
},
}));
};

const handleTableChange = (tableState: TableState<ProgramEvent>) => {
const { selection } = tableState;
onChange(syncRule.updateDataSyncEvents(selection.map(({ id }) => id)));
};

const updateSyncAll = (value: boolean) => {
onChange(syncRule.updateDataSyncAllEvents(value).updateDataSyncEvents(undefined));
};

const addToSelection = (ids: string[]) => {
const oldSelection = _.difference(syncRule.dataSyncEvents, ids);
const newSelection = _.difference(ids, syncRule.dataSyncEvents);

onChange(syncRule.updateDataSyncEvents([...oldSelection, ...newSelection]));
};
}, [programFilter, programs]);

const columns: TableColumn<ProgramEvent>[] = [
{ name: "id" as const, text: i18n.t("UID"), sortable: true },
{
name: "program" as const,
text: i18n.t("Program"),
sortable: true,
getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
},
{ name: "orgUnitName" as const, text: i18n.t("Organisation unit"), sortable: true },
{ name: "eventDate" as const, text: i18n.t("Event date"), sortable: true },
{
name: "lastUpdated" as const,
text: i18n.t("Last updated"),
sortable: true,
hidden: true,
},
{ name: "status" as const, text: i18n.t("Status"), sortable: true },
{ name: "storedBy" as const, text: i18n.t("Stored by"), sortable: true },
];

const details: ObjectsTableDetailField<ProgramEvent>[] = [
{ name: "id" as const, text: i18n.t("UID") },
{
name: "program" as const,
text: i18n.t("Program"),
getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
},
{ name: "orgUnitName" as const, text: i18n.t("Organisation unit") },
{ name: "created" as const, text: i18n.t("Created") },
{ name: "lastUpdated" as const, text: i18n.t("Last updated") },
{ name: "eventDate" as const, text: i18n.t("Event date") },
{ name: "dueDate" as const, text: i18n.t("Due date") },
{ name: "status" as const, text: i18n.t("Status") },
{ name: "storedBy" as const, text: i18n.t("Stored by") },
];

const actions = [
{
name: "select",
text: i18n.t("Select"),
primary: true,
multiple: true,
onClick: addToSelection,
isActive: () => false,
},
];

const filterComponents = (
<Dropdown
key={"program-filter"}
items={programs}
onValueChange={changeProgramFilter}
value={programFilter}
label={i18n.t("Program")}
/>
);

const additionalColumns = buildAdditionalColumns();
const filteredObjects =
objects?.filter(({ program }) => !programFilter || program === programFilter) ?? [];

Expand All @@ -149,6 +173,8 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt
);
}

console.log("loading", objects === undefined);

return (
<React.Fragment>
<Toggle
Expand Down

0 comments on commit 3228af9

Please sign in to comment.