Merge remote-tracking branch 'upstream/master'

adrianq · adrianq · commit 94f0340dbec3 · 2020-09-24T21:10:49.000+01:00
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "metadata-synchronization",
     "description": "Advanced metadata & data synchronization utility",
-    "version": "2.2.0",
+    "version": "2.2.1",
     "license": "GPL-3.0",
     "author": "EyeSeeTea team",
     "homepage": ".",
diff --git a/src/data/events/EventsD2ApiRepository.ts b/src/data/events/EventsD2ApiRepository.ts
@@ -10,7 +10,7 @@ import {
 } from "../../domain/synchronization/entities/SynchronizationResult";
 import { cleanObjectDefault, cleanOrgUnitPaths } from "../../domain/synchronization/utils";
 import { DataImportParams } from "../../types/d2";
-import { D2Api } from "../../types/d2-api";
+import { D2Api, Pager } from "../../types/d2-api";
 
 export class EventsD2ApiRepository implements EventsRepository {
     private api: D2Api;
@@ -24,31 +24,91 @@ export class EventsD2ApiRepository implements EventsRepository {
         programs: string[] = [],
         defaults: string[] = []
     ): Promise<ProgramEvent[]> {
-        const { period, orgUnitPaths = [], events = [], allEvents } = params;
-        const [startDate, endDate] = buildPeriodFromParams(params);
+        if (params.allEvents) return this.getAllEvents(params, programs, defaults);
+        else return this.getSpecificEvents(params, programs, defaults);
+    }
 
+    /**
+     * Design choices and heads-up:
+     *  - The events endpoint does not support multiple values for a given filter
+     *    meaning you cannot query for multiple programs or multiple orgUnits in
+     *    the same API call. Instead you need to query one by one
+     *  - Querying one by one is not performant, instead we query for all events
+     *    available in the instance and manually filter them in this method
+     *  - For big databases querying for all events available in a given instance
+     *    with paging=false makes the instance to eventually go offline
+     *  - Instead of disabling paging we traverse all the events by paginating all
+     *    the available pages so that we can filter them afterwards
+     */
+    private async getAllEvents(
+        params: DataSynchronizationParams,
+        programs: string[] = [],
+        defaults: string[] = []
+    ): Promise<ProgramEvent[]> {
         if (programs.length === 0) return [];
 
-        const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
+        const { period, orgUnitPaths = [] } = params;
+        const [startDate, endDate] = buildPeriodFromParams(params);
 
+        const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
         const result = [];
 
-        for (const program of programs) {
-            const { events: response } = (await this.api
-                .get("/events", {
-                    paging: false,
+        const fetchApi = async (program: string, page: number) => {
+            return this.api
+                .get<EventExportResult>("/events", {
+                    pageSize: 250,
+                    totalPages: true,
+                    page,
                     program,
                     startDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined,
                     endDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined,
                 })
-                .getData()) as { events: (ProgramEvent & { event: string })[] };
+                .getData();
+        };
+
+        for (const program of programs) {
+            const { events, pager } = await fetchApi(program, 1);
+            result.push(...events);
 
-            result.push(...response);
+            for (let page = 2; page <= pager.pageCount; page += 1) {
+                const { events } = await fetchApi(program, page);
+                result.push(...events);
+            }
+        }
+
+        return _(result)
+            .filter(({ orgUnit }) => orgUnits.includes(orgUnit))
+            .map(object => ({ ...object, id: object.event }))
+            .map(object => cleanObjectDefault(object, defaults))
+            .value();
+    }
+
+    private async getSpecificEvents(
+        params: DataSynchronizationParams,
+        programs: string[] = [],
+        defaults: string[] = []
+    ): Promise<ProgramEvent[]> {
+        const { orgUnitPaths = [], events: filter = [] } = params;
+        if (programs.length === 0 || filter.length === 0) return [];
+
+        const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
+        const result = [];
+
+        for (const program of programs) {
+            for (const ids of _.chunk(filter, 300)) {
+                const { events } = await this.api
+                    .get<EventExportResult>("/events", {
+                        paging: false,
+                        program,
+                        event: ids.join(";"),
+                    })
+                    .getData();
+                result.push(...events);
+            }
         }
 
         return _(result)
             .filter(({ orgUnit }) => orgUnits.includes(orgUnit))
-            .filter(({ event }) => (allEvents ? true : events.includes(event)))
             .map(object => ({ ...object, id: object.event }))
             .map(object => cleanObjectDefault(object, defaults))
             .value();
@@ -125,3 +185,8 @@ interface EventsPostResponse {
         }[];
     };
 }
+
+interface EventExportResult {
+    events: Array<ProgramEvent & { event: string }>;
+    pager: Pager;
+}
diff --git a/src/data/metadata/__tests__/integration/sync-events.spec.ts b/src/data/metadata/__tests__/integration/sync-events.spec.ts
@@ -93,6 +93,7 @@ describe("Sync metadata", () => {
         remote.get("/dataValueSets", async () => ({ dataValues: [] }));
 
         local.get("/events", async () => ({
+            pager: { page: 1, pageCount: 1, pageSize: 1, total: 1 },
             events: [
                 {
                     storedBy: "widp.admin",
@@ -125,6 +126,7 @@ describe("Sync metadata", () => {
         }));
 
         remote.get("/events", async () => ({
+            pager: { page: 1, pageCount: 1, pageSize: 1, total: 1 },
             events: [
                 {
                     storedBy: "widp.admin",
diff --git a/src/presentation/react/components/sync-wizard/data/EventsSelectionStep.tsx b/src/presentation/react/components/sync-wizard/data/EventsSelectionStep.tsx
@@ -1,10 +1,11 @@
 import { Typography } from "@material-ui/core";
 import { ObjectsTable, ObjectsTableDetailField, TableColumn, TableState } from "d2-ui-components";
 import _ from "lodash";
-import React, { useEffect, useState } from "react";
+import React, { useCallback, useEffect, useMemo, useState } from "react";
 import { ProgramEvent } from "../../../../../domain/events/entities/ProgramEvent";
 import { DataElement, Program } from "../../../../../domain/metadata/entities/MetadataEntities";
 import i18n from "../../../../../locales";
+import SyncRule from "../../../../../models/syncRule";
 import { useAppContext } from "../../../contexts/AppContext";
 import Dropdown from "../../dropdown/Dropdown";
 import { Toggle } from "../../toggle/Toggle";
@@ -20,30 +21,127 @@ type CustomProgram = Program & {
 
 export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardStepProps) {
     const { compositionRoot } = useAppContext();
+
+    const [memoizedSyncRule] = useState<SyncRule>(syncRule);
     const [objects, setObjects] = useState<ProgramEvent[] | undefined>();
     const [programs, setPrograms] = useState<CustomProgram[]>([]);
     const [programFilter, changeProgramFilter] = useState<string>("");
-    const [error, setError] = useState();
+    const [error, setError] = useState<unknown>();
+
+    useEffect(() => {
+        const sync = compositionRoot.sync.events(memoizedSyncRule.toBuilder());
+        sync.extractMetadata<CustomProgram>().then(({ programs = [] }) => setPrograms(programs));
+    }, [memoizedSyncRule, compositionRoot]);
 
     useEffect(() => {
+        if (programs.length === 0) return;
         compositionRoot.events
             .list(
                 {
-                    ...syncRule.dataParams,
+                    ...memoizedSyncRule.dataParams,
                     allEvents: true,
                 },
                 programs.map(({ id }) => id)
             )
             .then(setObjects)
             .catch(setError);
-    }, [compositionRoot, syncRule, programs]);
+    }, [compositionRoot, memoizedSyncRule, programs]);
 
-    useEffect(() => {
-        const sync = compositionRoot.sync.events(syncRule.toBuilder());
-        sync.extractMetadata<CustomProgram>().then(({ programs = [] }) => setPrograms(programs));
-    }, [syncRule, compositionRoot]);
+    const handleTableChange = useCallback(
+        (tableState: TableState<ProgramEvent>) => {
+            const { selection } = tableState;
+            onChange(syncRule.updateDataSyncEvents(selection.map(({ id }) => id)));
+        },
+        [onChange, syncRule]
+    );
+
+    const updateSyncAll = useCallback(
+        (value: boolean) => {
+            onChange(syncRule.updateDataSyncAllEvents(value).updateDataSyncEvents(undefined));
+        },
+        [onChange, syncRule]
+    );
+
+    const addToSelection = useCallback(
+        (ids: string[]) => {
+            const oldSelection = _.difference(syncRule.dataSyncEvents, ids);
+            const newSelection = _.difference(ids, syncRule.dataSyncEvents);
+
+            onChange(syncRule.updateDataSyncEvents([...oldSelection, ...newSelection]));
+        },
+        [onChange, syncRule]
+    );
+
+    const columns: TableColumn<ProgramEvent>[] = useMemo(
+        () => [
+            { name: "id" as const, text: i18n.t("UID"), sortable: true },
+            {
+                name: "program" as const,
+                text: i18n.t("Program"),
+                sortable: true,
+                getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
+            },
+            { name: "orgUnitName" as const, text: i18n.t("Organisation unit"), sortable: true },
+            { name: "eventDate" as const, text: i18n.t("Event date"), sortable: true },
+            {
+                name: "lastUpdated" as const,
+                text: i18n.t("Last updated"),
+                sortable: true,
+                hidden: true,
+            },
+            { name: "status" as const, text: i18n.t("Status"), sortable: true },
+            { name: "storedBy" as const, text: i18n.t("Stored by"), sortable: true },
+        ],
+        [programs]
+    );
+
+    const details: ObjectsTableDetailField<ProgramEvent>[] = useMemo(
+        () => [
+            { name: "id" as const, text: i18n.t("UID") },
+            {
+                name: "program" as const,
+                text: i18n.t("Program"),
+                getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
+            },
+            { name: "orgUnitName" as const, text: i18n.t("Organisation unit") },
+            { name: "created" as const, text: i18n.t("Created") },
+            { name: "lastUpdated" as const, text: i18n.t("Last updated") },
+            { name: "eventDate" as const, text: i18n.t("Event date") },
+            { name: "dueDate" as const, text: i18n.t("Due date") },
+            { name: "status" as const, text: i18n.t("Status") },
+            { name: "storedBy" as const, text: i18n.t("Stored by") },
+        ],
+        [programs]
+    );
+
+    const actions = useMemo(
+        () => [
+            {
+                name: "select",
+                text: i18n.t("Select"),
+                primary: true,
+                multiple: true,
+                onClick: addToSelection,
+                isActive: () => false,
+            },
+        ],
+        [addToSelection]
+    );
 
-    const buildAdditionalColumns = () => {
+    const filterComponents = useMemo(
+        () => (
+            <Dropdown
+                key={"program-filter"}
+                items={programs}
+                onValueChange={changeProgramFilter}
+                value={programFilter}
+                label={i18n.t("Program")}
+            />
+        ),
+        [programFilter, programs]
+    );
+
+    const additionalColumns = useMemo(() => {
         const program = _.find(programs, { id: programFilter });
         const dataElements = _(program?.programStages ?? [])
             .map(({ programStageDataElements }) =>
@@ -61,82 +159,8 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt
                 return _.find(row.dataValues, { dataElement: id })?.value ?? "-";
             },
         }));
-    };
-
-    const handleTableChange = (tableState: TableState<ProgramEvent>) => {
-        const { selection } = tableState;
-        onChange(syncRule.updateDataSyncEvents(selection.map(({ id }) => id)));
-    };
-
-    const updateSyncAll = (value: boolean) => {
-        onChange(syncRule.updateDataSyncAllEvents(value).updateDataSyncEvents(undefined));
-    };
-
-    const addToSelection = (ids: string[]) => {
-        const oldSelection = _.difference(syncRule.dataSyncEvents, ids);
-        const newSelection = _.difference(ids, syncRule.dataSyncEvents);
-
-        onChange(syncRule.updateDataSyncEvents([...oldSelection, ...newSelection]));
-    };
+    }, [programFilter, programs]);
 
-    const columns: TableColumn<ProgramEvent>[] = [
-        { name: "id" as const, text: i18n.t("UID"), sortable: true },
-        {
-            name: "program" as const,
-            text: i18n.t("Program"),
-            sortable: true,
-            getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
-        },
-        { name: "orgUnitName" as const, text: i18n.t("Organisation unit"), sortable: true },
-        { name: "eventDate" as const, text: i18n.t("Event date"), sortable: true },
-        {
-            name: "lastUpdated" as const,
-            text: i18n.t("Last updated"),
-            sortable: true,
-            hidden: true,
-        },
-        { name: "status" as const, text: i18n.t("Status"), sortable: true },
-        { name: "storedBy" as const, text: i18n.t("Stored by"), sortable: true },
-    ];
-
-    const details: ObjectsTableDetailField<ProgramEvent>[] = [
-        { name: "id" as const, text: i18n.t("UID") },
-        {
-            name: "program" as const,
-            text: i18n.t("Program"),
-            getValue: ({ program }) => _.find(programs, { id: program })?.name ?? program,
-        },
-        { name: "orgUnitName" as const, text: i18n.t("Organisation unit") },
-        { name: "created" as const, text: i18n.t("Created") },
-        { name: "lastUpdated" as const, text: i18n.t("Last updated") },
-        { name: "eventDate" as const, text: i18n.t("Event date") },
-        { name: "dueDate" as const, text: i18n.t("Due date") },
-        { name: "status" as const, text: i18n.t("Status") },
-        { name: "storedBy" as const, text: i18n.t("Stored by") },
-    ];
-
-    const actions = [
-        {
-            name: "select",
-            text: i18n.t("Select"),
-            primary: true,
-            multiple: true,
-            onClick: addToSelection,
-            isActive: () => false,
-        },
-    ];
-
-    const filterComponents = (
-        <Dropdown
-            key={"program-filter"}
-            items={programs}
-            onValueChange={changeProgramFilter}
-            value={programFilter}
-            label={i18n.t("Program")}
-        />
-    );
-
-    const additionalColumns = buildAdditionalColumns();
     const filteredObjects =
         objects?.filter(({ program }) => !programFilter || program === programFilter) ?? [];
 
@@ -149,6 +173,8 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt
         );
     }
 
+    console.log("loading", objects === undefined);
+
     return (
         <React.Fragment>
             <Toggle

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "metadata-synchronization",`
`3`	`3`	`"description": "Advanced metadata & data synchronization utility",`
`4`		`- "version": "2.2.0",`
	`4`	`+ "version": "2.2.1",`
`5`	`5`	`"license": "GPL-3.0",`
`6`	`6`	`"author": "EyeSeeTea team",`
`7`	`7`	`"homepage": ".",`
Original file line number	Diff line number	Diff line change
`@@ -93,6 +93,7 @@ describe("Sync metadata", () => {`
`93`	`93`	`remote.get("/dataValueSets", async () => ({ dataValues: [] }));`
`94`	`94`
`95`	`95`	`local.get("/events", async () => ({`
	`96`	`+ pager: { page: 1, pageCount: 1, pageSize: 1, total: 1 },`
`96`	`97`	`events: [`
`97`	`98`	`{`
`98`	`99`	`storedBy: "widp.admin",`
`@@ -125,6 +126,7 @@ describe("Sync metadata", () => {`
`125`	`126`	`}));`
`126`	`127`
`127`	`128`	`remote.get("/events", async () => ({`
	`129`	`+ pager: { page: 1, pageCount: 1, pageSize: 1, total: 1 },`
`128`	`130`	`events: [`
`129`	`131`	`{`
`130`	`132`	`storedBy: "widp.admin",`