Merge branch 'feat/migration-MT-update' into test-mt-dsfr

SocialGouv · Oct 18, 2024 · 3cda874 · 3cda874
2 parents 68c8baf + b7eb204
commit 3cda874
Show file tree

Hide file tree

Showing 9 changed files with 2,153 additions and 5,939 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,7 +1,6 @@
 name: Release
 on:
   schedule:
-    # * is a special character in YAML so you have to quote this string
     - cron: "00 21 * * *"
   repository_dispatch:
     types: manual_release

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,31 @@
+# [4.700.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.699.0...v4.700.0) (2024-09-24)
+
+
+### Features
+
+* disable scheduled released until new version ([#422](https://github.com/SocialGouv/fiches-travail-data/issues/422)) ([3a072b6](https://github.com/SocialGouv/fiches-travail-data/commit/3a072b612301caee576a8d8a16014001982948bc))
+
+# [4.699.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.698.0...v4.699.0) (2024-09-20)
+
+
+### Features
+
+* **data:** 20240920_2121 update ([0f16a9f](https://github.com/SocialGouv/fiches-travail-data/commit/0f16a9f015660ff8daa32e12b1a49673ede2f500))
+
+# [4.698.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.697.0...v4.698.0) (2024-09-19)
+
+
+### Features
+
+* **data:** 20240919_2120 update ([ccc4e65](https://github.com/SocialGouv/fiches-travail-data/commit/ccc4e65adf3ceb9d16155990b01fc0b278f2bb28))
+
+# [4.697.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.696.0...v4.697.0) (2024-09-17)
+
+
+### Features
+
+* **data:** 20240917_2121 update ([58020e9](https://github.com/SocialGouv/fiches-travail-data/commit/58020e93ae4f8e0b41180775d621ff43d4bedac6))
+
 # [4.696.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.695.0...v4.696.0) (2024-09-15)
 
 

diff --git a/data/fiches-travail.json b/data/fiches-travail.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@socialgouv/fiches-travail-data",
-  "version": "4.696.0",
+  "version": "4.700.0",
   "main": "build/index.js",
   "module": "build/index.js",
   "files": [

diff --git a/src/fetch-data/__tests__/__snapshots__/parseDom.test.ts.snap b/src/fetch-data/__tests__/__snapshots__/parseDom.test.ts.snap
diff --git a/src/fetch-data/__tests__/agents-chimiques-dangereux-acd.html b/src/fetch-data/__tests__/agents-chimiques-dangereux-acd.html
diff --git a/src/fetch-data/__tests__/article-complex-html.html b/src/fetch-data/__tests__/article-complex-html.html
diff --git a/src/fetch-data/__tests__/parseDom.test.ts b/src/fetch-data/__tests__/parseDom.test.ts
diff --git a/src/fetch-data/parseDom.js b/src/fetch-data/parseDom.js
@@ -136,6 +136,16 @@ export const textClean = (text, noNbsp = false) => {
     .trim();
 };
 
+const duplicateContent = (sections, highlight) => {
+  if (highlight) {
+    return (
+      sections.find((section) => highlight.text.includes(section.text)) !==
+      undefined
+    );
+  }
+  return false;
+};
+
 function parseHTMLSections(dom) {
   const document = dom.window.document;
 
@@ -180,7 +190,14 @@ function parseHTMLSections(dom) {
     sections.push(section);
   });
 
-  if (sections.find((section) => section.html === "")) {
+  const cleanSections = sections.map((section) => ({
+    ...section,
+    // Sometimes, we have all the html in a section
+    // We check a second times and delete HTML from the h2 found
+    // (H2 should not be in a section)
+    html: removeExtraH2(section.html),
+  }));
+  if (cleanSections.find((section) => section.html === "")) {
     return [
       {
         title: "Contenu",
@@ -189,13 +206,9 @@ function parseHTMLSections(dom) {
       },
     ];
   }
-  return sections.map((section) => ({
-    ...section,
-    // Sometimes, we have all the html in a section
-    // We check a second times and delete HTML from the h2 found
-    // (H2 should not be in a section)
-    html: removeExtraH2(section.html),
-  }));
+  if (cleanSections) {
+    return cleanSections;
+  }
 }
 
 const removeExtraH2 = (html) => {
@@ -329,6 +342,9 @@ export function parseDom(dom, id, url) {
   let sections = parseHTMLSections(dom);
 
   const highlight = parseHighlight(dom);
+  if (duplicateContent(sections, highlight)) {
+    sections = [];
+  }
   if (highlight) {
     sections.unshift(highlight);
   }