fix(data): don't save file over 100MB due to git limitation

SocialGouv · Sep 6, 2023 · 64a3e43 · 64a3e43
1 parent c01d034
commit 64a3e43
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 21 deletions.
diff --git a/.github/workflows/fetch-test.yml b/.github/workflows/fetch-test.yml
@@ -11,8 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: 'true'
 
       - name: Set up Node.js
         uses: actions/setup-node@v3

diff --git a/.github/workflows/fetch.yml b/.github/workflows/fetch.yml
@@ -10,8 +10,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          lfs: 'true'
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:

diff --git a/fetch.js b/fetch.js
@@ -40,25 +40,28 @@ const fetchAll = async () => {
     fs.mkdirSync(`./data/${type}`, { recursive: true });
     const fiches = await getDatasetJson(type, url);
     const writeSpinner = ora(`Writing "${type}" fiches`).start();
-    fiches.forEach((fiche) => {
-      const fileName = `./data/${type}/${fiche.id}.json`;
-      const fileContent = JSON.stringify(fiche, null, 2);
-      if (fileContent.length > 100000000) {
-        writeSpinner.warn(
-          `Error saving "${fileName}": Size is too big ${
-            fileContent.length / 1000000
-          }MB (git limitation is 100MB). If you need this file, please consider using git-lfs or compression.`
-        );
-      } else {
-        try {
-          fs.writeFileSync(fileName, fileContent);
-        } catch (err) {
-          writeSpinner.warn(`Error saving "${fileName}": ${err.message}`);
+    const fichesIdArray = fiches
+      .map((fiche) => {
+        const fileName = `./data/${type}/${fiche.id}.json`;
+        const fileContent = JSON.stringify(fiche, null, 2);
+        if (fileContent.length > 100000000) {
+          writeSpinner.warn(
+            `Error saving "${fileName}": Size is too big ${
+              fileContent.length / 1000000
+            }MB (git limitation is 100MB). If you need this file, please consider using git-lfs or compression.`
+          );
+          return undefined;
+        } else {
+          try {
+            fs.writeFileSync(fileName, fileContent);
+          } catch (err) {
+            writeSpinner.warn(`Error saving "${fileName}": ${err.message}`);
+          }
         }
-      }
-    });
+        return fiche.id;
+      })
+      .filter((ficheId) => ficheId !== undefined);
     const indexName = `./data/${type}/index.json`;
-    const fichesIdArray = fiches.map((fiche) => fiche.id);
     try {
       fs.writeFileSync(indexName, JSON.stringify(fichesIdArray, null, 2));
     } catch (err) {