MRG: Merge pull request #103 from octue/enhancement/allow-datasets-to…

…-be-optional

Allow optional datasets in manifest strands

setup.py

@@ -16,7 +16,7 @@
 
 setup(
     name="twined",
-    version="0.3.0",
+    version="0.3.1",
     py_modules=[],
     install_requires=["jsonschema ~= 4.4.0", "python-dotenv"],
     url="https://www.github.com/octue/twined",

tests/test_manifest_strands.py

@@ -82,6 +82,31 @@ def test_error_raised_if_datasets_are_missing_from_manifest(self):
             "A dataset named 'cat' is expected in the input_manifest but is missing.",
         )
 
+    def test_missing_optional_datasets_do_not_raise_error(self):
+        """Test that optional datasets specified in the twine missing from the manifest don't raise an error."""
+        twine = """
+            {
+                "input_manifest": {
+                    "datasets": {
+                        "cat": {
+                            "purpose": "blah",
+                            "optional": true
+                        },
+                        "dog": {
+                            "purpose": "blah"
+                        }
+                    }
+                }
+            }
+        """
+
+        input_manifest = {
+            "id": "30d2c75c-a7b9-4f16-8627-9c8d5cc04bf4",
+            "datasets": {"dog": "gs://dog-house/dog"},
+        }
+
+        Twine(source=twine).validate_input_manifest(source=input_manifest)
+
     def test_valid_manifest_files(self):
         """Ensures that a manifest file will validate."""
         valid_configuration_manifest = """

twined/schema/twine_schema.json

@@ -52,6 +52,9 @@
                 },
                 "file_tags_template": {
                   "$ref": "#/$defs/file_tags_template"
+                },
+                "optional": {
+                  "type": "boolean"
                 }
               }
             }

twined/twine.py

@@ -191,23 +191,26 @@ def _validate_manifest(self, kind, source, cls=None, **kwargs):
         return data
 
     def _validate_all_expected_datasets_are_present_in_manifest(self, manifest_kind, manifest):
-        """Check that all datasets specified in the corresponding manifest strand in the twine are present in the given
-        manifest.
+        """Check that all non-optional datasets specified in the corresponding manifest strand in the twine are present
+        in the given manifest.
 
         :param str manifest_kind: the kind of manifest that's being validated (so the correct schema can be accessed)
         :param dict manifest: the manifest whose datasets are to be validated
-        :raise twined.exceptions.InvalidManifestContents: if one or more of the expected datasets is missing
+        :raise twined.exceptions.InvalidManifestContents: if one or more of the expected non-optional datasets is missing
         :return None:
         """
-        # This is the manifest schema included in the twine.json file, not the schema for `manifest.json` files.
+        # This is the manifest schema included in the `twine.json` file, not the schema for `manifest.json` files.
         manifest_schema = getattr(self, manifest_kind)
 
-        for expected_dataset in manifest_schema["datasets"]:
-            if expected_dataset in manifest["datasets"]:
+        for expected_dataset_name, expected_dataset_schema in manifest_schema["datasets"].items():
+            if expected_dataset_name in manifest["datasets"]:
+                continue
+
+            if expected_dataset_schema.get("optional", False):
                 continue
 
             raise exceptions.invalid_contents_map[manifest_kind](
-                f"A dataset named {expected_dataset!r} is expected in the {manifest_kind} but is missing."
+                f"A dataset named {expected_dataset_name!r} is expected in the {manifest_kind} but is missing."
             )
 
     @property