Merge pull request #1053 from openzim/mindtouch

New changes for mindtouch
openzim · Nov 22, 2024 · d9a01c6 · d9a01c6
2 parents 18fdc87 + cc75b08
commit d9a01c6
Showing 1 changed file with 15 additions and 8 deletions.
diff --git a/dispatcher/backend/src/common/schemas/offliners/mindtouch.py b/dispatcher/backend/src/common/schemas/offliners/mindtouch.py
@@ -171,16 +171,23 @@ class Meta:
         metadata={"label": "Debug", "description": "Enable verbose output"},
     )
 
-    html_issues_warn_only = fields.Boolean(
-        truthy=[True],
-        falsy=[False],
+    bad_assets_regex = String(
+        metadata={
+            "label": "Bad assets regex",
+            "description": "Regular expression of asset URLs known to not be available."
+            "Case insensitive.",
+        },
+        data_key="bad-assets-regex",
+    )
+
+    bad_assets_threshold = fields.Integer(
         metadata={
-            "label": "HTML issues warn only",
-            "description": "[dev] Only log a warning when unexpected HTML is "
-            "encountered. Use with caution because activating this option means that "
-            "ZIM HTML will probably lead to online resources without user noticing it.",
+            "label": "Bad assets threshold",
+            "description": "[dev] Number of assets allowed to fail to download before "
+            "failing the scraper. Assets already excluded with --bad-assets-regex are "
+            "not counted for this threshold. Defaults to 10 assets.",
         },
-        data_key="html-issues-warn-only",
+        data_key="bad-assets-threshold",
     )
 
     stats_filename = String(