fix: external sync (#278)

* chore: create utils function to chunk array * fix: do not parallel sync for externalResources pb of heavy resources * bump: 2.3.3-rc.0 * chore: type callBack functions * refactor: typing and error handling on chunk function * test: add tests on chunk function --------- Co-authored-by: Quentin Ruhier <[email protected]>
InseeFr · Jan 8, 2025 · bc22110 · bc22110
1 parent 68eb561
commit bc22110
Show file tree

Hide file tree

Showing 9 changed files with 202 additions and 26 deletions.
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "drama-queen",
   "private": true,
-  "version": "2.3.2",
+  "version": "2.3.3-rc.0",
   "type": "module",
   "scripts": {
     "dev": "vite --port 5001 --strictPort",

diff --git a/src/core/tools/array.test.ts b/src/core/tools/array.test.ts
@@ -0,0 +1,50 @@
+import { describe, expect, it } from 'vitest'
+
+import { chunk } from './array'
+
+describe('chunk function', () => {
+  it('splits an array into chunks of the specified size', () => {
+    const array = [1, 2, 3, 4, 5]
+    const result = chunk(array, 2)
+    expect(result).toEqual([[1, 2], [3, 4], [5]])
+  })
+
+  it('returns an empty array if input array is empty', () => {
+    const result = chunk([], 3)
+    expect(result).toEqual([])
+  })
+
+  it('throws an error if chunkSize is not strictly positive', () => {
+    const array = [1, 2, 3]
+    expect(() => chunk(array, 0)).toThrow(
+      'chunkSize must be a strictly positive integer',
+    )
+    expect(() => chunk(array, -2)).toThrow(
+      'chunkSize must be a strictly positive integer',
+    )
+    expect(() => chunk(array, 0.5)).toThrow(
+      'chunkSize must be a strictly positive integer',
+    )
+  })
+
+  it('handles chunkSize larger than the array length', () => {
+    const array = [1, 2, 3]
+    const result = chunk(array, 10)
+    expect(result).toEqual([[1, 2, 3]])
+  })
+
+  it('works with arrays of objects', () => {
+    const array = [{ id: 1 }, { id: 2 }, { id: 3 }]
+    const result = chunk(array, 2)
+    expect(result).toEqual([[{ id: 1 }, { id: 2 }], [{ id: 3 }]])
+  })
+
+  it('works with mixed-type arrays', () => {
+    const array = [1, 'a', true, null]
+    const result = chunk(array, 2)
+    expect(result).toEqual([
+      [1, 'a'],
+      [true, null],
+    ])
+  })
+})
diff --git a/src/core/tools/array.ts b/src/core/tools/array.ts
@@ -0,0 +1,12 @@
+export const chunk = <T>(array: T[], chunkSize: number) => {
+  if (chunkSize <= 0 || !Number.isInteger(chunkSize)) {
+    throw new Error('chunkSize must be a strictly positive integer')
+  }
+
+  const res: T[][] = []
+  for (let i = 0; i < array.length; i += chunkSize) {
+    const chunk = array.slice(i, i + chunkSize)
+    res.push(chunk)
+  }
+  return res
+}
diff --git a/src/core/tools/externalResources.ts b/src/core/tools/externalResources.ts
@@ -7,6 +7,7 @@ import type {
   Manifest,
 } from '@/core/model'
 
+import { chunk } from './array'
 import { fetchUrl } from './fetchUrl'
 
 const EXTERNAL_QUESTIONNAIRES_KEYWORD = 'gide'
@@ -64,18 +65,44 @@ export async function filterTransformedManifest(
 }
 
 // Cache every external resources (not already cached) for a particular questionnaire
-export async function getResourcesFromExternalQuestionnaire(
-  questionnaire: ExternalQuestionnaire,
-): Promise<void> {
+export async function getResourcesFromExternalQuestionnaire({
+  questionnaire,
+  callBackTotal,
+  callBackReset,
+  callBackUnit,
+}: {
+  questionnaire: ExternalQuestionnaire
+  callBackTotal: (total: number) => void
+  callBackReset: () => void
+  callBackUnit: () => void
+}): Promise<void> {
   const transformedManifest = await getTransformedManifest(questionnaire.id)
 
   const filteredTransformedManifest = await filterTransformedManifest(
     questionnaire.cacheName,
     transformedManifest,
   )
 
-  const manifestCache = await caches.open(questionnaire.cacheName)
-  return await manifestCache.addAll(filteredTransformedManifest)
+  const transformManifestFilteredChunked = chunk<string>(
+    filteredTransformedManifest,
+    15,
+  )
+  callBackReset()
+  callBackTotal(transformManifestFilteredChunked.length)
+
+  return await (transformManifestFilteredChunked || []).reduce(
+    async (previousPromise, subManifest) => {
+      await previousPromise
+
+      const putSubManifestInCache = async () => {
+        const cacheForManifest = await caches.open(questionnaire.cacheName)
+        await cacheForManifest.addAll(subManifest)
+        callBackUnit()
+      }
+      return putSubManifestInCache()
+    },
+    Promise.resolve(),
+  )
 }
 
 // Separate, from the list of external questionnaires, those that are needed and those that are not needed

diff --git a/src/core/usecases/synchronizeData/selectors.ts b/src/core/usecases/synchronizeData/selectors.ts
@@ -47,17 +47,42 @@ const externalResourcesProgress = createSelector(downloadingState, (state) => {
     return undefined
   }
   // if there is no external resources, we don't show the progress bar
-  if (state.totalExternalResources === undefined) {
+  if (state.totalExternalResourcesByQuestionnaire === undefined) {
     return undefined
   }
   if (
-    state.externalResourcesCompleted === 0 &&
-    state.totalExternalResources === 0
+    state.externalResourcesByQuestionnaireCompleted === 0 &&
+    state.totalExternalResourcesByQuestionnaire === 0
   )
     return 100
-  return (state.externalResourcesCompleted * 100) / state.totalExternalResources
+  return (
+    (state.externalResourcesByQuestionnaireCompleted * 100) /
+    state.totalExternalResourcesByQuestionnaire
+  )
 })
 
+const externalResourcesProgressCount = createSelector(
+  downloadingState,
+  (state) => {
+    if (state === undefined) {
+      return undefined
+    }
+    // if there is no external resources, we don't show the progress bar
+    if (state.totalExternalResources === undefined) {
+      return undefined
+    }
+    if (
+      state.totalExternalResources === 0 &&
+      state.externalResourcesCompleted === 0
+    )
+      return undefined
+    return {
+      externalResourcesCompleted: state.externalResourcesCompleted,
+      totalExternalResources: state.totalExternalResources,
+    }
+  },
+)
+
 const uploadProgress = createSelector(state, (state) => {
   if (state.stateDescription !== 'running') {
     return undefined
@@ -77,13 +102,15 @@ const main = createSelector(
   nomenclatureProgress,
   surveyProgress,
   externalResourcesProgress,
+  externalResourcesProgressCount,
   uploadProgress,
   (
     state,
     surveyUnitProgress,
     nomenclatureProgress,
     surveyProgress,
     externalResourcesProgress,
+    externalResourcesProgressCount,
     uploadProgress,
   ) => {
     switch (state.stateDescription) {
@@ -107,6 +134,7 @@ const main = createSelector(
               nomenclatureProgress,
               surveyProgress,
               externalResourcesProgress,
+              externalResourcesProgressCount,
             }
         }
     }

diff --git a/src/core/usecases/synchronizeData/state.ts b/src/core/usecases/synchronizeData/state.ts
@@ -32,6 +32,8 @@ export namespace State {
       nomenclatureCompleted: number
       totalSurvey: number
       surveyCompleted: number
+      totalExternalResourcesByQuestionnaire?: number
+      externalResourcesByQuestionnaireCompleted: number
       totalExternalResources?: number
       externalResourcesCompleted: number
     }
@@ -61,6 +63,10 @@ export const { reducer, actions } = createUsecaseActions({
           // for total external resources, we make difference for displaying progress bar between :
           // 0 : external synchro is triggered but there is no needed questionnaire so we want a fullfilled progress bar
           // undefined : external synchro is not triggered so we don't want the progress bar
+          totalExternalResourcesByQuestionnaire: EXTERNAL_RESOURCES_URL
+            ? Infinity
+            : undefined,
+          externalResourcesByQuestionnaireCompleted: 0,
           totalExternalResources: EXTERNAL_RESOURCES_URL ? Infinity : undefined,
           externalResourcesCompleted: 0,
         }),
@@ -136,13 +142,41 @@ export const { reducer, actions } = createUsecaseActions({
         totalExternalResources,
       }
     },
-    downloadExternalResourceCompleted: (state) => {
+    setDownloadExternalResourcesCompleted: (state) => {
       assert(state.stateDescription === 'running' && state.type === 'download')
       return {
         ...state,
         externalResourcesCompleted: state.externalResourcesCompleted + 1,
       }
     },
+    setDownloadTotalExternalResourcesByQuestionnaire: (
+      state,
+      {
+        payload,
+      }: { payload: { totalExternalResourcesByQuestionnaire: number } },
+    ) => {
+      const { totalExternalResourcesByQuestionnaire } = payload
+      assert(state.stateDescription === 'running' && state.type === 'download')
+      return {
+        ...state,
+        totalExternalResourcesByQuestionnaire,
+      }
+    },
+    downloadExternalResourceByQuestionnaireCompleted: (state) => {
+      assert(state.stateDescription === 'running' && state.type === 'download')
+      return {
+        ...state,
+        externalResourcesByQuestionnaireCompleted:
+          state.externalResourcesByQuestionnaireCompleted + 1,
+      }
+    },
+    downloadExternalResourceReset: (state) => {
+      assert(state.stateDescription === 'running' && state.type === 'download')
+      return {
+        ...state,
+        externalResourcesByQuestionnaireCompleted: 0,
+      }
+    },
     setUploadTotal: (state, { payload }: { payload: { total: number } }) => {
       const { total } = payload
       assert(state.stateDescription === 'running' && state.type === 'upload')

diff --git a/src/core/usecases/synchronizeData/thunks.ts b/src/core/usecases/synchronizeData/thunks.ts
@@ -219,19 +219,36 @@ export const thunks = {
           )
 
           // add in cache the missing external resources for needed questionnaires
-          const prGetExternalResources = Promise.all(
-            neededQuestionnaires.map(async (questionnaire) => {
-              await getResourcesFromExternalQuestionnaire(questionnaire)
-                .then(() =>
-                  dispatch(actions.downloadExternalResourceCompleted()),
-                )
+          const prGetExternalResources = (neededQuestionnaires || []).reduce(
+            async (previousPromise, questionnaire) => {
+              await previousPromise
+
+              return getResourcesFromExternalQuestionnaire({
+                questionnaire: questionnaire,
+                callBackTotal: (total: number) =>
+                  dispatch(
+                    actions.setDownloadTotalExternalResourcesByQuestionnaire({
+                      totalExternalResourcesByQuestionnaire: total,
+                    }),
+                  ),
+                callBackReset: () =>
+                  dispatch(actions.downloadExternalResourceReset()),
+                callBackUnit: () =>
+                  dispatch(
+                    actions.downloadExternalResourceByQuestionnaireCompleted(),
+                  ),
+              })
+                .then(() => {
+                  dispatch(actions.setDownloadExternalResourcesCompleted())
+                })
                 .catch((error) =>
                   console.error(
                     `An error occurred while fetching external resources of questionnaire ${questionnaire.id}`,
                     error,
                   ),
                 )
-            }),
+            },
+            Promise.resolve(),
           )
 
           // delete the cache of every not needed external questionnaires

diff --git a/src/ui/pages/synchronize/LoadingDisplay.tsx b/src/ui/pages/synchronize/LoadingDisplay.tsx
@@ -11,7 +11,8 @@ type LoadingDisplayProps = {
   syncStepTitle: string
   progressBars: {
     progress: number
-    label: string | undefined
+    label?: string
+    extraTitle?: string
   }[]
 }
 
@@ -30,21 +31,22 @@ export function LoadingDisplay(props: LoadingDisplayProps) {
         </Typography>
       </Stack>
       <Stack spacing={2}>
-        {progressBars.map((bar) => (
-          <Fragment key={`${bar.label}-${bar.progress}`}>
+        {progressBars.map(({ label, progress, extraTitle }) => (
+          <Fragment key={`${label}-${progress}`}>
             <Stack spacing={1}>
-              {bar.label !== undefined && (
+              {label !== undefined && (
                 <Typography
                   variant="body2"
                   fontWeight="bold"
                   className={classes.lightText}
                 >
-                  {bar.label}
+                  {label}
+                  {extraTitle ? `: ${extraTitle}` : ''}
                 </Typography>
               )}
               <LinearProgress
                 variant="determinate"
-                value={bar.progress}
+                value={progress}
                 className={classes.progressBar}
               />
             </Stack>

diff --git a/src/ui/pages/synchronize/SynchronizeData.tsx b/src/ui/pages/synchronize/SynchronizeData.tsx
@@ -18,6 +18,7 @@ export function SynchronizeData() {
     surveyProgress,
     surveyUnitProgress,
     externalResourcesProgress,
+    externalResourcesProgressCount,
     uploadProgress,
   } = useCoreState('synchronizeData', 'main')
 
@@ -51,7 +52,6 @@ export function SynchronizeData() {
           progressBars={[
             {
               progress: uploadProgress,
-              label: undefined,
             },
           ]}
           syncStepTitle={t('uploadingData')}
@@ -73,11 +73,17 @@ export function SynchronizeData() {
               label: t('surveyUnitsProgress'),
             },
             // render external resources progress bar only if there are external resources
-            ...(externalResourcesProgress !== undefined
+            ...(externalResourcesProgress !== undefined &&
+            externalResourcesProgressCount !== undefined
               ? [
                   {
                     progress: externalResourcesProgress,
                     label: t('externalResourcesProgress'),
+                    extraTitle: Number.isFinite(
+                      externalResourcesProgressCount.totalExternalResources,
+                    )
+                      ? `${externalResourcesProgressCount.externalResourcesCompleted} / ${externalResourcesProgressCount.totalExternalResources}`
+                      : undefined,
                   },
                 ]
               : []),