From 12d5aaede144596eb8815a723e0f57cbfaeef386 Mon Sep 17 00:00:00 2001 From: Gift Nnko Date: Thu, 20 Jan 2022 09:53:08 +0300 Subject: [PATCH 1/2] fix(Migration): Fixes - Fixed issues with automatic migration - Added skip migration check --- src/core/constants/migration.js | 3 + src/core/state/scorecard.js | 23 +- .../hooks/useMigrateScorecard.js | 213 ++++++------------ .../ScorecardMigration/hooks/useQueue.js | 31 +++ .../Components/ScorecardMigration/index.js | 18 +- .../ScorecardMigration/services/migrate.js | 72 ++++++ src/modules/Main/hooks/autoMigration.js | 15 ++ src/modules/Main/index.js | 14 +- src/modules/Router/index.js | 106 +++++---- 9 files changed, 278 insertions(+), 217 deletions(-) create mode 100644 src/core/constants/migration.js create mode 100644 src/modules/Main/Components/ScorecardMigration/hooks/useQueue.js create mode 100644 src/modules/Main/Components/ScorecardMigration/services/migrate.js create mode 100644 src/modules/Main/hooks/autoMigration.js diff --git a/src/core/constants/migration.js b/src/core/constants/migration.js new file mode 100644 index 000000000..8ab602e9a --- /dev/null +++ b/src/core/constants/migration.js @@ -0,0 +1,3 @@ + + +export const DATA_MIGRATION_CHECK ="skipMigration" diff --git a/src/core/state/scorecard.js b/src/core/state/scorecard.js index 2e2a35445..ae6ca58cc 100644 --- a/src/core/state/scorecard.js +++ b/src/core/state/scorecard.js @@ -75,18 +75,28 @@ const ScorecardIdState = atom({ default: null, }); + +const AllScorecardsSummaryState = selector({ + key: "all-scorecard-summary-state", + get: async ({get}) => { + const engine = get(EngineState); + await restoreScorecardSummary(engine); + const {summary, error} = await getScorecardSummary(engine); + if (error) { + throw error; + } + return summary; + } +}) + + const ScorecardSummaryState = atom({ key: "scorecard-summary", default: selector({ key: "scorecard-summary-selector", get: async ({get}) => { - const engine = get(EngineState); + const summary = get(AllScorecardsSummaryState); const user = get(UserState); - await restoreScorecardSummary(engine); - const {summary, error} = await getScorecardSummary(engine); - if (error) { - throw error; - } return filter(summary, (scorecardSummary) => { const {read} = getUserAuthority(user, scorecardSummary) ?? {}; return read; @@ -381,4 +391,5 @@ export { ScorecardDataSourceState, ScorecardDataLoadingState, ScorecardLegendDefinitionSelector, + AllScorecardsSummaryState }; diff --git a/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js b/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js index 8206de9b8..cbfad87f1 100644 --- a/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js +++ b/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js @@ -1,159 +1,86 @@ -import { useDataEngine } from "@dhis2/app-runtime"; -import { queue } from "async"; +import {useDataEngine} from "@dhis2/app-runtime"; +import {useSetting} from "@dhis2/app-service-datastore"; +import {compact, filter, isEmpty, map, uniqBy} from "lodash"; +import {useCallback, useEffect, useState} from "react"; +import {useRecoilRefresher_UNSTABLE, useRecoilValue} from "recoil"; +import {DATA_MIGRATION_CHECK} from "../../../../../core/constants/migration"; +import {AllScorecardsSummaryState} from "../../../../../core/state/scorecard"; +import {migrateScorecard} from "../../../../../shared/utils/migrate"; +import {generateScorecardSummary} from "../../../../../shared/utils/scorecard"; import { - compact, - differenceBy, - forIn, - fromPairs, - isEmpty, - uniqBy, -} from "lodash"; -import { useEffect, useState } from "react"; -import { useResetRecoilState } from "recoil"; -import { - DATASTORE_ENDPOINT, - DATASTORE_OLD_SCORECARD_ENDPOINT, - DATASTORE_SCORECARD_SUMMARY_KEY, -} from "../../../../../core/constants/config"; -import { ScorecardSummaryState } from "../../../../../core/state/scorecard"; -import getScorecardSummary from "../../../../../shared/services/getScorecardSummary"; -import { migrateScorecard } from "../../../../../shared/utils/migrate"; -import { - generateCreateMutation, - generateScorecardSummary, -} from "../../../../../shared/utils/scorecard"; - -const oldScorecardsQuery = { - scorecardKeys: { - resource: DATASTORE_OLD_SCORECARD_ENDPOINT, - }, -}; - -const generateOldScorecardQueries = (ids = []) => { - return fromPairs( - ids?.map((id) => [ - id, - { - resource: DATASTORE_OLD_SCORECARD_ENDPOINT, - id, - }, - ]) - ); -}; + getOldScorecardKeys, + getOldScorecards, + getScorecardKeys, + uploadNewScorecard, + uploadSummary +} from "../services/migrate"; +import useQueue from "./useQueue"; -async function getOldScorecards(engine) { - const keys = await engine.query(oldScorecardsQuery); - if (isEmpty(keys?.scorecardKeys)) { - return []; - } - const oldScorecardsObject = await engine.query( - generateOldScorecardQueries(keys?.scorecardKeys) - ); - const oldScorecardsList = []; - forIn(oldScorecardsObject, (value, key) => { - oldScorecardsList.push({ ...value, id: key }); - }); - - return oldScorecardsList; -} +export default function useMigrateScorecard(onComplete) { + const [error, setError] = useState(); + const allSummary = useRecoilValue(AllScorecardsSummaryState) + const resetSummary = useRecoilRefresher_UNSTABLE(AllScorecardsSummaryState); + const [summaries, setSummaries] = useState(); + const engine = useDataEngine(); + const [, { set: setSkipMigration }] = useSetting(DATA_MIGRATION_CHECK, { global: true }); -const uploadNewScorecard = async ({ scorecard, engine }) => { - const newScorecard = migrateScorecard(scorecard); - if (newScorecard) { - return await engine.mutate(generateCreateMutation(newScorecard?.id), { - variables: { data: newScorecard }, - }); - } -}; -const summaryMutation = { - type: "update", - resource: DATASTORE_ENDPOINT, - id: DATASTORE_SCORECARD_SUMMARY_KEY, - data: ({ data }) => data, -}; + const migrate = useCallback( + async (scorecard) => { + await uploadNewScorecard({newScorecard: scorecard, engine}) + }, + [engine], + ); -const uploadSummary = async (engine, summary) => { - return await engine.mutate(summaryMutation, { variables: { data: summary } }); -}; + const onMigrationComplete = useCallback(async () => { + await uploadSummary(engine, uniqBy([...allSummary, ...summaries], 'id')) + resetSummary(); + setSkipMigration(true); + onComplete() + }, [allSummary, engine, onComplete, resetSummary, summaries]) -const q = queue(uploadNewScorecard); + const {add, progress, length, started} = useQueue({ + drain: onMigrationComplete, + task: migrate + }) -export default function useMigrateScorecard(onComplete) { - const [error, setError] = useState(); - const [loading, setLoading] = useState(false); - const resetSummary = useResetRecoilState(ScorecardSummaryState); - const [progress, setProgress] = useState(0); - const [count, setCount] = useState(0); - const engine = useDataEngine(); - useEffect(() => { - async function migrate() { - setLoading(true); - try { - const { summary, error } = await getScorecardSummary(engine); - if (error) { - throw error; - } - const oldScorecards = await getOldScorecards(engine); - setLoading(false); - if (!isEmpty(oldScorecards)) { - const unMigratedScorecards = differenceBy( - oldScorecards, - summary, - "id" - ); - if (isEmpty(unMigratedScorecards)) { - onComplete(); - return; - } - setCount(unMigratedScorecards.length); - q.push( - unMigratedScorecards?.map((scorecard) => ({ scorecard, engine })), - () => { - setProgress((prevState) => prevState + 1); - } - ); - q.drain(async () => { - const newSummary = unMigratedScorecards?.map((oldScorecard) => { - const newScorecard = migrateScorecard(oldScorecard); - return generateScorecardSummary(newScorecard); + const onMigrationInitiated = useCallback(async () => { + try { + const scorecardKeys = await getScorecardKeys(engine); + const oldScorecardKeys = await getOldScorecardKeys(engine); + const filteredKeys = filter(oldScorecardKeys, (key) => { + return !scorecardKeys.includes(key); }); - if (!isEmpty(compact(newSummary))) { - const allSummary = uniqBy([...summary, ...newSummary], "id"); - await uploadSummary(engine, allSummary).then(() => { - resetSummary(); - onComplete(); - }); + if (filteredKeys && !isEmpty(filteredKeys)) { + const oldScorecards = compact(await getOldScorecards(engine, filteredKeys)); + const newScorecards = compact(map(oldScorecards, migrateScorecard)); + const newScorecardsSummaries = compact(map(newScorecards, generateScorecardSummary)) + setSummaries(newScorecardsSummaries); + for (const scorecard of newScorecards) { + add(scorecard) + } } else { - onComplete(); + onComplete(); + setSkipMigration(true) } - }); - } else { - onComplete(); + } catch (e) { + setError(e); + onComplete() } - } catch (e) { - if (e?.details?.httpStatusCode === 404) { - onComplete(); - return; - } - if (e?.details?.httpStatusCode === 403) { - onComplete(); - return; - } - setError(e); - onComplete(); - } - } - migrate(); - }, []); + }, [add, engine, onComplete]) + + + useEffect(() => { + onMigrationInitiated(); + }, []); - return { - progress, - count, - loading, - error, - }; + return { + progress, + count: progress + length, + error, + migrationStarted: started + }; } diff --git a/src/modules/Main/Components/ScorecardMigration/hooks/useQueue.js b/src/modules/Main/Components/ScorecardMigration/hooks/useQueue.js new file mode 100644 index 000000000..03f484cd4 --- /dev/null +++ b/src/modules/Main/Components/ScorecardMigration/hooks/useQueue.js @@ -0,0 +1,31 @@ +import async from "async"; +import { useCallback, useRef, useState } from "react"; + +export default function useQueue({ drain, task }) { + const [progress, setProgress] = useState(0); + const queue = useRef( + async.queue((variable, callback) => { + task(variable).then((results) => { + callback(results); + }); + }, 1), + ); + queue.current?.drain(drain); + + const add = useCallback((task) => { + queue.current.push(task, (err) => { + if (err) { + console.error(err); + } + setProgress((prevState) => prevState + 1); + }); + }, []); + + return { + add, + started: queue.current?.started, + kill: queue.current?.kill, + progress, + length: queue.current?.length(), + }; +} diff --git a/src/modules/Main/Components/ScorecardMigration/index.js b/src/modules/Main/Components/ScorecardMigration/index.js index 219703837..3ce505194 100644 --- a/src/modules/Main/Components/ScorecardMigration/index.js +++ b/src/modules/Main/Components/ScorecardMigration/index.js @@ -1,21 +1,21 @@ import {useAlert} from "@dhis2/app-runtime"; import i18n from "@dhis2/d2-i18n"; import {CircularLoader, LinearLoader} from "@dhis2/ui"; -import PropTypes from "prop-types"; import React, {useEffect} from "react"; +import {useHistory} from "react-router-dom"; import useMigrateScorecard from "./hooks/useMigrateScorecard"; -export default function ScorecardMigration({onMigrationComplete}) { +export default function ScorecardMigration() { + const history = useHistory(); const onComplete = () => { - onMigrationComplete(true); + history.replace("/"); }; const {show} = useAlert( ({message}) => message, ({type}) => ({...type, duration: 3000}) ); - const {loading, error, progress, count} = useMigrateScorecard(onComplete); - + const {error, progress, count, migrationStarted} = useMigrateScorecard(onComplete); useEffect(() => { if (error) { show({ @@ -25,10 +25,11 @@ export default function ScorecardMigration({onMigrationComplete}) { } }, [error, show]); - if (loading) { + if (!migrationStarted) { return (
- + +

{i18n.t("Preparing migration...")}

); } @@ -43,6 +44,3 @@ export default function ScorecardMigration({onMigrationComplete}) { ); } -ScorecardMigration.propTypes = { - onMigrationComplete: PropTypes.func.isRequired, -}; diff --git a/src/modules/Main/Components/ScorecardMigration/services/migrate.js b/src/modules/Main/Components/ScorecardMigration/services/migrate.js new file mode 100644 index 000000000..869bf79a2 --- /dev/null +++ b/src/modules/Main/Components/ScorecardMigration/services/migrate.js @@ -0,0 +1,72 @@ +import {filter, forIn, fromPairs, isEmpty} from "lodash"; +import { + DATASTORE_ENDPOINT, + DATASTORE_OLD_SCORECARD_ENDPOINT, + DATASTORE_SCORECARD_SUMMARY_KEY +} from "../../../../../core/constants/config"; +import {generateCreateMutation} from "../../../../../shared/utils/scorecard"; + +const generateOldScorecardQueries = (ids = []) => { + return fromPairs( + ids?.map((id) => [ + id, + { + resource: DATASTORE_OLD_SCORECARD_ENDPOINT, + id, + }, + ]) + ); +}; + +export async function getOldScorecards(engine, keys) { + if (isEmpty(keys)) { + return []; + } + const oldScorecardsObject = await engine.query( + generateOldScorecardQueries(keys) + ); + const oldScorecardsList = []; + forIn(oldScorecardsObject, (value, key) => { + oldScorecardsList.push({...value, id: key}); + }); + + return oldScorecardsList; +} + +export const uploadNewScorecard = async ({newScorecard, engine}) => { + if (newScorecard) { + return await engine.mutate(generateCreateMutation(newScorecard?.id), { + variables: {data: newScorecard}, + }); + } +}; + + +export async function getOldScorecardKeys(engine) { + const {keys} = await engine.query({ + keys: { + resource: DATASTORE_OLD_SCORECARD_ENDPOINT, + } + }) ?? {}; + return keys; +} + +export async function getScorecardKeys(engine) { + const {keys} = await engine.query({ + keys: { + resource: DATASTORE_ENDPOINT, + } + }) ?? {}; + return filter(keys, (key) => !(key.includes(DATASTORE_SCORECARD_SUMMARY_KEY) || key.includes("settings") || key.includes("savedObjects"))); +} + +const summaryMutation = { + type: "update", + resource: DATASTORE_ENDPOINT, + id: DATASTORE_SCORECARD_SUMMARY_KEY, + data: ({data}) => data, +}; + +export const uploadSummary = async (engine, summary) => { + return await engine.mutate(summaryMutation, {variables: {data: summary}}); +}; diff --git a/src/modules/Main/hooks/autoMigration.js b/src/modules/Main/hooks/autoMigration.js new file mode 100644 index 000000000..eada7ec20 --- /dev/null +++ b/src/modules/Main/hooks/autoMigration.js @@ -0,0 +1,15 @@ +import {useSetting} from "@dhis2/app-service-datastore"; +import {useEffect} from "react"; +import {useHistory} from "react-router-dom"; +import {DATA_MIGRATION_CHECK} from "../../../core/constants/migration"; + +export function useAutoMigration() { + const [skipMigration] = useSetting(DATA_MIGRATION_CHECK, {global: true}); + const history = useHistory(); + + useEffect(() => { + if (!skipMigration) { + history.replace("/migrate"); + } + }, [history, skipMigration]); +} diff --git a/src/modules/Main/index.js b/src/modules/Main/index.js index 17a2b2f49..968e72c5e 100644 --- a/src/modules/Main/index.js +++ b/src/modules/Main/index.js @@ -1,12 +1,10 @@ -import React, { useState } from "react"; +import React from "react"; import ScorecardList from "./Components/ScorecardList"; -import ScorecardMigration from "./Components/ScorecardMigration"; +import {useAutoMigration} from "./hooks/autoMigration"; export default function Main() { - const [migrationComplete, setMigrationComplete] = useState(false); - return migrationComplete ? ( - - ) : ( - - ); + useAutoMigration(); + return ( + + ) } diff --git a/src/modules/Router/index.js b/src/modules/Router/index.js index adfa84f4e..fe1827103 100644 --- a/src/modules/Router/index.js +++ b/src/modules/Router/index.js @@ -1,63 +1,69 @@ -import React, { Suspense } from "react"; -import { ErrorBoundary } from "react-error-boundary"; -import { HashRouter, Redirect, Route, Switch } from "react-router-dom"; -import { useRecoilValue } from "recoil"; -import { SystemSettingsState } from "../../core/state/system"; +import React, {Suspense} from "react"; +import {ErrorBoundary} from "react-error-boundary"; +import {HashRouter, Redirect, Route, Switch} from "react-router-dom"; +import {useRecoilValue} from "recoil"; +import {SystemSettingsState} from "../../core/state/system"; import FullPageError from "../../shared/Components/Errors/FullPageError"; -import { FullPageLoader } from "../../shared/Components/Loaders"; +import {FullPageLoader} from "../../shared/Components/Loaders"; const Main = React.lazy(() => import("../Main")); const ScorecardManagement = React.lazy(() => - import("../Main/Components/ScoreCardManagement") + import("../Main/Components/ScoreCardManagement") ); const ScorecardView = React.lazy(() => - import("../Main/Components/ScorecardView") + import("../Main/Components/ScorecardView") +); + +const ScorecardMigration = React.lazy(() => + import("../Main/Components/ScorecardMigration") ); const pages = [ - { - pathname: "/edit/:id", - component: ScorecardManagement, - }, - { - pathname: "/add", - component: ScorecardManagement, - }, - { - pathname: "/view/:id", - component: ScorecardView, - }, - { - pathname: "/", - component: Main, - }, + { + pathname: "/migrate", + component: ScorecardMigration, + }, + { + pathname: "/edit/:id", + component: ScorecardManagement, + }, + { + pathname: "/add", + component: ScorecardManagement, + }, + { + pathname: "/view/:id", + component: ScorecardView, + }, + { + pathname: "/", + component: Main, + }, ]; export default function Router() { - useRecoilValue(SystemSettingsState); - - - return ( - - - }> - - {pages.map(({ pathname, component }) => { - const Component = component; - return ( - - - - - - ); - })} - - - - - - - - ); + useRecoilValue(SystemSettingsState); + return ( + + + }> + + {pages.map(({pathname, component}) => { + const Component = component; + return ( + + + + + + ); + })} + + + + + + + + ); } From 92fc80392043dc6bdbbcbe1f14b881f46b6886e7 Mon Sep 17 00:00:00 2001 From: Gift Nnko Date: Thu, 20 Jan 2022 10:49:57 +0300 Subject: [PATCH 2/2] fix(Migration): Fixes - Fixed organisation unit assignment on migration --- .../hooks/useMigrateScorecard.js | 2 +- src/shared/utils/migrate.js | 24 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js b/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js index cbfad87f1..c8e8f5709 100644 --- a/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js +++ b/src/modules/Main/Components/ScorecardMigration/hooks/useMigrateScorecard.js @@ -38,7 +38,7 @@ export default function useMigrateScorecard(onComplete) { resetSummary(); setSkipMigration(true); onComplete() - }, [allSummary, engine, onComplete, resetSummary, summaries]) + }, [allSummary, engine, onComplete, resetSummary, setSkipMigration, summaries]) const {add, progress, length, started} = useQueue({ drain: onMigrationComplete, diff --git a/src/shared/utils/migrate.js b/src/shared/utils/migrate.js index 015d84f9c..93f3c7668 100644 --- a/src/shared/utils/migrate.js +++ b/src/shared/utils/migrate.js @@ -1,4 +1,4 @@ -import { find, has } from "lodash"; +import {find, has, isEmpty} from "lodash"; import { uid } from "./utils"; export function migrateScorecard(oldScorecard) { @@ -19,7 +19,8 @@ export function migrateScorecard(oldScorecard) { ), targetOnLevels: false, periodSelection: getScorecardPeriodSelection( - oldScorecard.selected_periods + oldScorecard.selected_periods, + oldScorecard.periodType ), orgUnitSelection: getScorecardOrgUnitSelection( oldScorecard.orgunit_settings @@ -65,8 +66,11 @@ function getScorecardLegendDefinitions(oldScorecardLegendDefinitions) { }); } -function getScorecardPeriodSelection(oldScorecardPeriodSelections) { - return {}; +function getScorecardPeriodSelection(oldScorecardPeriodSelections, periodType) { + return { + periods: oldScorecardPeriodSelections.map(period=> ({id: period.id, name: period.name})), + type: periodType + }; } function getScorecardDataSelection(oldScorecardDataSelections) { @@ -126,15 +130,15 @@ function getScorecardDataSource(indicator) { function getScorecardOrgUnitSelection(oldScorecardOrgUnitSelections) { return { - groups: oldScorecardOrgUnitSelections.selected_groups || [], - levels: oldScorecardOrgUnitSelections.selected_levels || [], - orgUnits: oldScorecardOrgUnitSelections.selected_orgunits.map( + groups: oldScorecardOrgUnitSelections?.selected_groups?.map(group=> group?.id) || [], + levels: oldScorecardOrgUnitSelections?.selected_levels?.map(level=>level?.id) || [], + orgUnits: oldScorecardOrgUnitSelections?.selected_orgunits?.map( (selectedOrgUnit) => ({ id: selectedOrgUnit.id }) ), - userOrgUnit: false, - userSubUnit: false, - userSubX2Unit: false, + userOrgUnit: Boolean(find(oldScorecardOrgUnitSelections?.selected_user_orgunit, ["id", "USER_ORGUNIT"])), + userSubUnit: Boolean(find(oldScorecardOrgUnitSelections?.selected_user_orgunit, ["id", "USER_ORGUNIT_CHILDREN"])), + userSubX2Unit: Boolean(find(oldScorecardOrgUnitSelections?.selected_user_orgunit, ["id", "USER_ORGUNIT_GRANDCHILDREN"])), }; }