From 7200b6d8d91dbc0587960ceed68dc1b573de0731 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 20 Nov 2020 21:43:15 +0000 Subject: [PATCH 01/13] VER Bump for 0.0.15 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b76b9d..868653f 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="twined", - version="0.0.14", + version="0.0.15", py_modules=[], install_requires=["jsonschema ~= 3.2.0", "python-dotenv"], url="https://www.github.com/octue/twined", From ba77658b4ce898d729e734c64bd9e6d527acc53e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 16 Dec 2020 11:15:10 +0000 Subject: [PATCH 02/13] CHO: Add version consistency workflow --- .github/workflows/check-version-consistency.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/check-version-consistency.yml diff --git a/.github/workflows/check-version-consistency.yml b/.github/workflows/check-version-consistency.yml new file mode 100644 index 0000000..a3c4c01 --- /dev/null +++ b/.github/workflows/check-version-consistency.yml @@ -0,0 +1,11 @@ +name: Check release branch version against setup.py + +on: [push] + +jobs: + check-version-consistency: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python .github/workflows/scripts/check-version-consistency.py From 31a62db549951733d74a360fe95f0cb970d65467 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 16 Dec 2020 15:45:28 +0000 Subject: [PATCH 03/13] FIX: Add missing script --- .../scripts/check-version-consistency.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/scripts/check-version-consistency.py diff --git a/.github/workflows/scripts/check-version-consistency.py b/.github/workflows/scripts/check-version-consistency.py new file mode 100644 index 0000000..62ae22d --- /dev/null +++ b/.github/workflows/scripts/check-version-consistency.py @@ -0,0 +1,68 @@ +import os +import subprocess +import sys + + +PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + + +class NotAVersionBranchException(Exception): + pass + + +class InvalidBranchNameFormat(Exception): + pass + + +def get_setup_version(): + process = subprocess.run(["python", "setup.py", "--version"], capture_output=True) + return process.stdout.strip().decode("utf8") + + +def get_branch_name(): + process = subprocess.run(["git", "branch", "--show-current"], capture_output=True) + return process.stdout.strip().decode("utf8") + + +def release_branch_version_matches_setup_version(setup_version, full_branch_name): + """ Check if the package version stated in setup.py matches the semantic version 'x.y.z' included in the branch name + of the format 'release/x.y.z'. + + :param str setup_version: + :param str full_branch_name: + :raise NotAVersionBranchException: + :return bool: + """ + try: + branch_type, branch_name = full_branch_name.split("/") + except ValueError: + raise InvalidBranchNameFormat( + f"The branch name must be in the form 'branch_type/branch_name'; received {full_branch_name!r}" + ) + + if branch_type != "release": + raise NotAVersionBranchException(f"The branch is not a release branch: {full_branch_name!r}.") + + return branch_name == setup_version + + +if __name__ == "__main__": + + os.chdir(PACKAGE_ROOT) + setup_version = get_setup_version() + full_branch_name = get_branch_name() + + try: + if release_branch_version_matches_setup_version(setup_version, full_branch_name): + print(f"Release branch name matches setup.py version: {setup_version!r}.") + sys.exit(0) + + print( + f"Release branch name does not match setup.py version: branch is {full_branch_name!r} but setup.py version " + f"is {setup_version!r}." + ) + sys.exit(1) + + except NotAVersionBranchException as e: + print(e.args[0]) + sys.exit(0) From 95ccdb85a657f08de7e8e254bad410cb8f8bb06a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 23 Dec 2020 11:03:21 +0000 Subject: [PATCH 04/13] OPS: Move release version check into python-ci --- .github/workflows/check-version-consistency.yml | 11 ----------- .github/workflows/python-ci.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 .github/workflows/check-version-consistency.yml diff --git a/.github/workflows/check-version-consistency.yml b/.github/workflows/check-version-consistency.yml deleted file mode 100644 index a3c4c01..0000000 --- a/.github/workflows/check-version-consistency.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Check release branch version against setup.py - -on: [push] - -jobs: - check-version-consistency: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: python .github/workflows/scripts/check-version-consistency.py diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 64ffe9c..9582311 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -9,6 +9,14 @@ name: python-ci on: [push] jobs: + + check-version-consistency: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python .github/workflows/scripts/check-version-consistency.py + tests: runs-on: ubuntu-latest env: From 7f853da67342811eece90a9e31081ab93e26afa0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Jan 2021 13:28:16 +0000 Subject: [PATCH 05/13] FIX: Make strand attributes public; add missing argument to Twine.validate_children --- tests/test_children.py | 4 ++-- twined/twine.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_children.py b/tests/test_children.py index 21a6287..7335514 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -37,13 +37,13 @@ def test_valid_children(self): "children": [{"key": "gis", "purpose": "The purpose.", "notes": "Some notes.", "filters": "tags:gis"}] } """ - self.assertEqual(len(Twine(source=source)._children), 1) + self.assertEqual(len(Twine(source=source).children), 1) def test_empty_children(self): """ Ensures that a twine file will validate with an empty list object as children """ twine = Twine(source="""{"children": []}""") - self.assertEqual(len(twine._children), 0) + self.assertEqual(len(twine.children), 0) class TestChildrenValidation(BaseTestCase): diff --git a/twined/twine.py b/twined/twine.py index 0ef56fe..608077c 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -54,7 +54,7 @@ def __init__(self, **kwargs): """ Constructor for the twine class """ for name, strand in self._load_twine(**kwargs).items(): - setattr(self, "_" + name, strand) + setattr(self, name, strand) self._available_strands = tuple(trim_suffix(name, "_schema") for name in vars(self)) @@ -119,7 +119,7 @@ def _validate_against_schema(self, strand, data): else: if strand not in SCHEMA_STRANDS: raise exceptions.UnknownStrand(f"Unknown strand {strand}. Try one of {ALL_STRANDS}.") - schema_key = "_" + strand + "_schema" + schema_key = strand + "_schema" try: schema = getattr(self, schema_key) except AttributeError: @@ -178,14 +178,14 @@ def available_strands(self): """ return self._available_strands - def validate_children(self, **kwargs): + def validate_children(self, source): """ Validates that the children values, passed as either a file or a json string, are correct """ # TODO cache this loaded data keyed on a hashed version of kwargs - children = self._load_json("children", **kwargs) + children = self._load_json("children", source) self._validate_against_schema("children", children) - strand = getattr(self, "_children", []) + strand = getattr(self, "children", []) # Loop the children and accumulate values so we have an O(1) check children_keys = {} @@ -249,7 +249,7 @@ def validate_credentials(self, dotenv_path=None): # Loop through the required credentials to check for presence of each credentials = {} - for credential in getattr(self, "_credentials", []): + for credential in getattr(self, "credentials", []): name = credential["name"] default = credential.get("default", None) credentials[name] = os.environ.get(name, default) From c2aba0001b208755a02264a35914aac66595998c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 6 Jan 2021 13:35:08 +0000 Subject: [PATCH 06/13] FIX: Restore **kwargs argument in Twine.validate_children --- twined/twine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twined/twine.py b/twined/twine.py index 608077c..2ac0d88 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -178,11 +178,11 @@ def available_strands(self): """ return self._available_strands - def validate_children(self, source): + def validate_children(self, source, **kwargs): """ Validates that the children values, passed as either a file or a json string, are correct """ # TODO cache this loaded data keyed on a hashed version of kwargs - children = self._load_json("children", source) + children = self._load_json("children", source, **kwargs) self._validate_against_schema("children", children) strand = getattr(self, "children", []) From 97298d8be49bcfab9fd8a83d61e7ed21f313486b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 20 Jan 2021 14:15:10 +0000 Subject: [PATCH 07/13] IMP: Only support Google pub/sub children --- twined/schema/children_schema.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 6f84192..22c6acd 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -11,12 +11,11 @@ "description": "The universally unique ID of the running child twin", "type": "string" }, - "uri_env_name": { - "description": "Name of the environment variable containing the URI of the twin (which may contain credentials)", - "type": "string", - "pattern": "^[A-Z]+(?:_[A-Z]+)*$" + "gcp_project_name": { + "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", + "type": "string" } }, - "required": ["key", "id", "uri_env_name"] + "required": ["key", "id", "gcp_project_name"] } } From a64aaaa82998aa60d9f3f2e69938ecd85bf66611 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 20 Jan 2021 21:50:30 +0000 Subject: [PATCH 08/13] DOC: Update schema description --- twined/schema/children_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 22c6acd..fd0638b 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -8,7 +8,7 @@ "type": "string" }, "id": { - "description": "The universally unique ID of the running child twin", + "description": "The universally unique ID (UUID) of the running child twin", "type": "string" }, "gcp_project_name": { From f7f8c7d7f1da50a340b47a0129a55b4fd94c7ecd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 25 Jan 2021 16:18:56 +0000 Subject: [PATCH 09/13] IMP: Allow different backends for children --- twined/schema/children_schema.json | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index fd0638b..88c3d05 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -11,11 +11,31 @@ "description": "The universally unique ID (UUID) of the running child twin", "type": "string" }, - "gcp_project_name": { - "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", - "type": "string" + "backend": { + "description": "The backend running the child.", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "GCP Pub/Sub", + "properties": { + "name": { + "description": "Type of backend (in this case, it should be GCPPubSub", + "type": "string" + }, + "project_name": { + "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", + "type": "string" + }, + "credentials_filename": { + "description": "Absolute path to Google Cloud Platform credentials JSON file.", + "type": "string" + } + } + } + ] } }, - "required": ["key", "id", "gcp_project_name"] + "required": ["key", "id", "backend"] } } From f6ffc277fc33c7e110ad8b25386e21be535ca9eb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:04:08 +0000 Subject: [PATCH 10/13] IMP: Only allow valid backend names in twine files --- twined/schema/children_schema.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 88c3d05..2ea7f79 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -20,8 +20,9 @@ "title": "GCP Pub/Sub", "properties": { "name": { - "description": "Type of backend (in this case, it should be GCPPubSub", - "type": "string" + "description": "Type of backend (in this case, it can only be GCPPubSub)", + "type": "string", + "pattern": "^(GCPPubSubBackend)$" }, "project_name": { "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", From 6b73b15b36386bcb3cfc84077fbec5fcafe0b276 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:09:32 +0000 Subject: [PATCH 11/13] TST: Fix children tests --- tests/test_children.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_children.py b/tests/test_children.py index 7335514..60263cf 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -58,7 +58,15 @@ class TestChildrenValidation(BaseTestCase): VALID_CHILD_VALUE = """ [ - {"key": "gis", "id": "some-id", "uri_env_name": "NAME_OF_SOME_ENV_VAR_THAT_CONTAINS_A_URI"} + { + "key": "gis", + "id": "some-id", + "backend": { + "name": "GCPPubSubBackend", + "project_name": "my-project", + "credentials_filename": "hello.json" + } + } ] """ @@ -101,7 +109,11 @@ def test_extra_key_validation_on_valid_twine(self): { "key": "gis", "id": "some-id", - "uri_env_name": "SOME_ENV_VAR_NAME", + "backend": { + "name": "GCPPubSubBackend", + "project_name": "my-project", + "credentials_filename": "hello.json" + }, "some_extra_property": "should not be a problem if present" } ] From f8ab34b5d202980f6d30eb7748c82be12b292453 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:14:47 +0000 Subject: [PATCH 12/13] FIX: Ensure the backend field of a child cannot be empty --- tests/test_children.py | 7 +++++++ twined/schema/children_schema.json | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_children.py b/tests/test_children.py index 60263cf..ecacf56 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -87,6 +87,13 @@ def test_extra_children(self): with self.assertRaises(exceptions.InvalidValuesContents): Twine().validate_children(source=self.VALID_CHILD_VALUE) + def test_backend_cannot_be_empty(self): + """ Test that the backend field of a child cannot be empty. """ + single_child_missing_backend = """[{"key": "gis", "id": "some-id", "backend": {}}]""" + + with self.assertRaises(exceptions.InvalidValuesContents): + Twine().validate_children(source=single_child_missing_backend) + def test_extra_key_validation_on_empty_twine(self): """ Test that children with extra data will not raise a validation error on an empty twine. """ diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index 2ea7f79..abe6114 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -32,7 +32,8 @@ "description": "Absolute path to Google Cloud Platform credentials JSON file.", "type": "string" } - } + }, + "required": ["name", "project_name", "credentials_filename"] } ] } From 536dee53197eecef7d02b22d155391b685cff63a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 26 Jan 2021 16:22:37 +0000 Subject: [PATCH 13/13] DOC: Correct field description in schema --- twined/schema/children_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json index abe6114..4aa0368 100644 --- a/twined/schema/children_schema.json +++ b/twined/schema/children_schema.json @@ -20,7 +20,7 @@ "title": "GCP Pub/Sub", "properties": { "name": { - "description": "Type of backend (in this case, it can only be GCPPubSub)", + "description": "Type of backend (in this case, it can only be GCPPubSubBackend)", "type": "string", "pattern": "^(GCPPubSubBackend)$" },