diff --git a/setup.py b/setup.py index a9012bd..c720dad 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="twined", - version="0.3.0", + version="0.3.1", py_modules=[], install_requires=["jsonschema ~= 4.4.0", "python-dotenv"], url="https://www.github.com/octue/twined", diff --git a/tests/test_manifest_strands.py b/tests/test_manifest_strands.py index a089069..63a3568 100644 --- a/tests/test_manifest_strands.py +++ b/tests/test_manifest_strands.py @@ -82,6 +82,31 @@ def test_error_raised_if_datasets_are_missing_from_manifest(self): "A dataset named 'cat' is expected in the input_manifest but is missing.", ) + def test_missing_optional_datasets_do_not_raise_error(self): + """Test that optional datasets specified in the twine missing from the manifest don't raise an error.""" + twine = """ + { + "input_manifest": { + "datasets": { + "cat": { + "purpose": "blah", + "optional": true + }, + "dog": { + "purpose": "blah" + } + } + } + } + """ + + input_manifest = { + "id": "30d2c75c-a7b9-4f16-8627-9c8d5cc04bf4", + "datasets": {"dog": "gs://dog-house/dog"}, + } + + Twine(source=twine).validate_input_manifest(source=input_manifest) + def test_valid_manifest_files(self): """Ensures that a manifest file will validate.""" valid_configuration_manifest = """ diff --git a/twined/schema/twine_schema.json b/twined/schema/twine_schema.json index e63d8f6..0974502 100644 --- a/twined/schema/twine_schema.json +++ b/twined/schema/twine_schema.json @@ -52,6 +52,9 @@ }, "file_tags_template": { "$ref": "#/$defs/file_tags_template" + }, + "optional": { + "type": "boolean" } } } diff --git a/twined/twine.py b/twined/twine.py index 081b026..be4d143 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -191,23 +191,26 @@ def _validate_manifest(self, kind, source, cls=None, **kwargs): return data def _validate_all_expected_datasets_are_present_in_manifest(self, manifest_kind, manifest): - """Check that all datasets specified in the corresponding manifest strand in the twine are present in the given - manifest. + """Check that all non-optional datasets specified in the corresponding manifest strand in the twine are present + in the given manifest. :param str manifest_kind: the kind of manifest that's being validated (so the correct schema can be accessed) :param dict manifest: the manifest whose datasets are to be validated - :raise twined.exceptions.InvalidManifestContents: if one or more of the expected datasets is missing + :raise twined.exceptions.InvalidManifestContents: if one or more of the expected non-optional datasets is missing :return None: """ - # This is the manifest schema included in the twine.json file, not the schema for `manifest.json` files. + # This is the manifest schema included in the `twine.json` file, not the schema for `manifest.json` files. manifest_schema = getattr(self, manifest_kind) - for expected_dataset in manifest_schema["datasets"]: - if expected_dataset in manifest["datasets"]: + for expected_dataset_name, expected_dataset_schema in manifest_schema["datasets"].items(): + if expected_dataset_name in manifest["datasets"]: + continue + + if expected_dataset_schema.get("optional", False): continue raise exceptions.invalid_contents_map[manifest_kind]( - f"A dataset named {expected_dataset!r} is expected in the {manifest_kind} but is missing." + f"A dataset named {expected_dataset_name!r} is expected in the {manifest_kind} but is missing." ) @property