From 7f853da67342811eece90a9e31081ab93e26afa0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Jan 2021 13:28:16 +0000 Subject: [PATCH 1/9] FIX: Make strand attributes public; add missing argument to Twine.validate_children --- tests/test_children.py | 4 ++-- twined/twine.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_children.py b/tests/test_children.py index 21a6287..7335514 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -37,13 +37,13 @@ def test_valid_children(self): "children": [{"key": "gis", "purpose": "The purpose.", "notes": "Some notes.", "filters": "tags:gis"}] } """ - self.assertEqual(len(Twine(source=source)._children), 1) + self.assertEqual(len(Twine(source=source).children), 1) def test_empty_children(self): """ Ensures that a twine file will validate with an empty list object as children """ twine = Twine(source="""{"children": []}""") - self.assertEqual(len(twine._children), 0) + self.assertEqual(len(twine.children), 0) class TestChildrenValidation(BaseTestCase): diff --git a/twined/twine.py b/twined/twine.py index 0ef56fe..608077c 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -54,7 +54,7 @@ def __init__(self, **kwargs): """ Constructor for the twine class """ for name, strand in self._load_twine(**kwargs).items(): - setattr(self, "_" + name, strand) + setattr(self, name, strand) self._available_strands = tuple(trim_suffix(name, "_schema") for name in vars(self)) @@ -119,7 +119,7 @@ def _validate_against_schema(self, strand, data): else: if strand not in SCHEMA_STRANDS: raise exceptions.UnknownStrand(f"Unknown strand {strand}. Try one of {ALL_STRANDS}.") - schema_key = "_" + strand + "_schema" + schema_key = strand + "_schema" try: schema = getattr(self, schema_key) except AttributeError: @@ -178,14 +178,14 @@ def available_strands(self): """ return self._available_strands - def validate_children(self, **kwargs): + def validate_children(self, source): """ Validates that the children values, passed as either a file or a json string, are correct """ # TODO cache this loaded data keyed on a hashed version of kwargs - children = self._load_json("children", **kwargs) + children = self._load_json("children", source) self._validate_against_schema("children", children) - strand = getattr(self, "_children", []) + strand = getattr(self, "children", []) # Loop the children and accumulate values so we have an O(1) check children_keys = {} @@ -249,7 +249,7 @@ def validate_credentials(self, dotenv_path=None): # Loop through the required credentials to check for presence of each credentials = {} - for credential in getattr(self, "_credentials", []): + for credential in getattr(self, "credentials", []): name = credential["name"] default = credential.get("default", None) credentials[name] = os.environ.get(name, default) From c2aba0001b208755a02264a35914aac66595998c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Jan 2021 13:35:08 +0000 Subject: [PATCH 2/9] FIX: Restore **kwargs argument in Twine.validate_children --- twined/twine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twined/twine.py b/twined/twine.py index 608077c..2ac0d88 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -178,11 +178,11 @@ def available_strands(self): """ return self._available_strands - def validate_children(self, source): + def validate_children(self, source, **kwargs): """ Validates that the children values, passed as either a file or a json string, are correct """ # TODO cache this loaded data keyed on a hashed version of kwargs - children = self._load_json("children", source) + children = self._load_json("children", source, **kwargs) self._validate_against_schema("children", children) strand = getattr(self, "children", []) From 97298d8be49bcfab9fd8a83d61e7ed21f313486b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 20 Jan 2021 14:15:10 +0000 Subject: [PATCH 3/9] IMP: Only support Google pub/sub children --- twined/schema/children_schema.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 6f84192..22c6acd 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -11,12 +11,11 @@ "description": "The universally unique ID of the running child twin", "type": "string" }, - "uri_env_name": { - "description": "Name of the environment variable containing the URI of the twin (which may contain credentials)", - "type": "string", - "pattern": "^[A-Z]+(?:_[A-Z]+)*$" + "gcp_project_name": { + "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", + "type": "string" } }, - "required": ["key", "id", "uri_env_name"] + "required": ["key", "id", "gcp_project_name"] } } From a64aaaa82998aa60d9f3f2e69938ecd85bf66611 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 20 Jan 2021 21:50:30 +0000 Subject: [PATCH 4/9] DOC: Update schema description --- twined/schema/children_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 22c6acd..fd0638b 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -8,7 +8,7 @@ "type": "string" }, "id": { - "description": "The universally unique ID of the running child twin", + "description": "The universally unique ID (UUID) of the running child twin", "type": "string" }, "gcp_project_name": { From f7f8c7d7f1da50a340b47a0129a55b4fd94c7ecd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 25 Jan 2021 16:18:56 +0000 Subject: [PATCH 5/9] IMP: Allow different backends for children --- twined/schema/children_schema.json | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index fd0638b..88c3d05 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -11,11 +11,31 @@ "description": "The universally unique ID (UUID) of the running child twin", "type": "string" }, - "gcp_project_name": { - "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", - "type": "string" + "backend": { + "description": "The backend running the child.", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "GCP Pub/Sub", + "properties": { + "name": { + "description": "Type of backend (in this case, it should be GCPPubSub", + "type": "string" + }, + "project_name": { + "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", + "type": "string" + }, + "credentials_filename": { + "description": "Absolute path to Google Cloud Platform credentials JSON file.", + "type": "string" + } + } + } + ] } }, - "required": ["key", "id", "gcp_project_name"] + "required": ["key", "id", "backend"] } } From f6ffc277fc33c7e110ad8b25386e21be535ca9eb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:04:08 +0000 Subject: [PATCH 6/9] IMP: Only allow valid backend names in twine files --- twined/schema/children_schema.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 88c3d05..2ea7f79 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -20,8 +20,9 @@ "title": "GCP Pub/Sub", "properties": { "name": { - "description": "Type of backend (in this case, it should be GCPPubSub", - "type": "string" + "description": "Type of backend (in this case, it can only be GCPPubSub)", + "type": "string", + "pattern": "^(GCPPubSubBackend)$" }, "project_name": { "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", From 6b73b15b36386bcb3cfc84077fbec5fcafe0b276 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:09:32 +0000 Subject: [PATCH 7/9] TST: Fix children tests --- tests/test_children.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_children.py b/tests/test_children.py index 7335514..60263cf 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -58,7 +58,15 @@ class TestChildrenValidation(BaseTestCase): VALID_CHILD_VALUE = """ [ - {"key": "gis", "id": "some-id", "uri_env_name": "NAME_OF_SOME_ENV_VAR_THAT_CONTAINS_A_URI"} + { + "key": "gis", + "id": "some-id", + "backend": { + "name": "GCPPubSubBackend", + "project_name": "my-project", + "credentials_filename": "hello.json" + } + } ] """ @@ -101,7 +109,11 @@ def test_extra_key_validation_on_valid_twine(self): { "key": "gis", "id": "some-id", - "uri_env_name": "SOME_ENV_VAR_NAME", + "backend": { + "name": "GCPPubSubBackend", + "project_name": "my-project", + "credentials_filename": "hello.json" + }, "some_extra_property": "should not be a problem if present" } ] From f8ab34b5d202980f6d30eb7748c82be12b292453 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:14:47 +0000 Subject: [PATCH 8/9] FIX: Ensure the backend field of a child cannot be empty --- tests/test_children.py | 7 +++++++ twined/schema/children_schema.json | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_children.py b/tests/test_children.py index 60263cf..ecacf56 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -87,6 +87,13 @@ def test_extra_children(self): with self.assertRaises(exceptions.InvalidValuesContents): Twine().validate_children(source=self.VALID_CHILD_VALUE) + def test_backend_cannot_be_empty(self): + """ Test that the backend field of a child cannot be empty. """ + single_child_missing_backend = """[{"key": "gis", "id": "some-id", "backend": {}}]""" + + with self.assertRaises(exceptions.InvalidValuesContents): + Twine().validate_children(source=single_child_missing_backend) + def test_extra_key_validation_on_empty_twine(self): """ Test that children with extra data will not raise a validation error on an empty twine. """ diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 2ea7f79..abe6114 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -32,7 +32,8 @@ "description": "Absolute path to Google Cloud Platform credentials JSON file.", "type": "string" } - } + }, + "required": ["name", "project_name", "credentials_filename"] } ] } From 536dee53197eecef7d02b22d155391b685cff63a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:22:37 +0000 Subject: [PATCH 9/9] DOC: Correct field description in schema --- twined/schema/children_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index abe6114..4aa0368 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -20,7 +20,7 @@ "title": "GCP Pub/Sub", "properties": { "name": { - "description": "Type of backend (in this case, it can only be GCPPubSub)", + "description": "Type of backend (in this case, it can only be GCPPubSubBackend)", "type": "string", "pattern": "^(GCPPubSubBackend)$" },