From a9a1014a8a0b11fd3f7a3c2295051c1852081441 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 22 Nov 2021 19:56:51 +0000 Subject: [PATCH 1/8] FEA: Add ability to validate monitor updates --- setup.py | 2 +- tests/test_monitors.py | 30 ++++++++++++++++++++++++++++++ twined/exceptions.py | 5 +++++ twined/twine.py | 12 ++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/test_monitors.py diff --git a/setup.py b/setup.py index 9bff06d..5fa88e3 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="twined", - version="0.0.26", + version="0.1.0", py_modules=[], install_requires=["jsonschema ~= 3.2.0", "python-dotenv"], url="https://www.github.com/octue/twined", diff --git a/tests/test_monitors.py b/tests/test_monitors.py new file mode 100644 index 0000000..e45a45e --- /dev/null +++ b/tests/test_monitors.py @@ -0,0 +1,30 @@ +from twined import Twine, exceptions +from .base import BaseTestCase + + +class TestMonitorsTwine(BaseTestCase): + STRAND_WITH_MONITORS_SCHEMA = """ + { + "monitors": { + "type": "object", + "properties": { + "my_property": { + "type": "number" + } + }, + "required": ["my_property"] + } + } + """ + + def test_validate_monitors_raises_error_if_monitors_schema_not_met(self): + """Test that an error is raised if an invalid monitor update is validated.""" + twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) + + with self.assertRaises(exceptions.InvalidMonitorsUpdate): + twine.validate_monitor_update([]) + + def test_validate_monitors_with_valid_monitor_update(self): + """Test that a valid monitor update validates successfully.""" + twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) + twine.validate_monitor_update({"my_property": 3.7}) diff --git a/twined/exceptions.py b/twined/exceptions.py index fb03716..98e01a6 100644 --- a/twined/exceptions.py +++ b/twined/exceptions.py @@ -105,6 +105,10 @@ class InvalidValuesContents(InvalidValues, ValidationError): """Raised when the JSON in the file is not valid according to its matching schema.""" +class InvalidMonitorsUpdate(InvalidValues, ValidationError): + pass + + # --------------------- Exceptions relating to validation of manifests ------------------------ @@ -171,4 +175,5 @@ class UnexpectedNumberOfResults(TwineException): "configuration_manifest": InvalidManifestContents, "input_manifest": InvalidManifestContents, "output_manifest": InvalidManifestContents, + "monitors": InvalidMonitorsUpdate, } diff --git a/twined/twine.py b/twined/twine.py index 9ccb7e3..c7c0804 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -330,6 +330,18 @@ def validate_output_manifest(self, source, **kwargs): """Validates the output manifest, passed as either a file or a json string""" return self._validate_manifest("output_manifest", source, **kwargs) + def validate_monitor_update(self, source): + """Validate a monitor update against the monitors strand.""" + strand = "monitors" + data = self._load_json("monitor_update", source) + + try: + jsonschema_validate(instance=data, schema=self.monitors) + logger.debug("Validated %s against schema.", strand) + + except ValidationError as e: + raise exceptions.invalid_contents_map[strand](str(e)) + @staticmethod def _get_cls(name, cls): """Getter that will return cls[name] if cls is a dict or cls otherwise""" From db7dcafc49616149f710a0078d1b97e9256211c6 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 22 Nov 2021 19:58:24 +0000 Subject: [PATCH 2/8] OPS: Use latest conventional commits pre-commit version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 384aa1e..65a2d61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: - '^release/(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' - repo: https://github.com/octue/pre-commit-hooks - rev: 0.0.3 + rev: 0.5.0 hooks: - id: check-commit-message-is-conventional stages: [commit-msg] From 1217277442da0a398dc2db01638ad7839c65e376 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Nov 2021 10:37:07 +0000 Subject: [PATCH 3/8] REF: Simplify monitors validation --- tests/test_monitors.py | 2 +- twined/twine.py | 21 ++++----------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/tests/test_monitors.py b/tests/test_monitors.py index e45a45e..254dc9b 100644 --- a/tests/test_monitors.py +++ b/tests/test_monitors.py @@ -5,7 +5,7 @@ class TestMonitorsTwine(BaseTestCase): STRAND_WITH_MONITORS_SCHEMA = """ { - "monitors": { + "monitors_schema": { "type": "object", "properties": { "my_property": { diff --git a/twined/twine.py b/twined/twine.py index c7c0804..10275be 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -12,11 +12,7 @@ logger = logging.getLogger(__name__) -SCHEMA_STRANDS = ( - "input_values", - "configuration_values", - "output_values", -) +SCHEMA_STRANDS = ("input_values", "configuration_values", "output_values", "monitors") MANIFEST_STRANDS = ( "configuration_manifest", @@ -28,14 +24,11 @@ CHILDREN_STRANDS = ("children",) -MONITOR_STRANDS = ("monitors",) - ALL_STRANDS = ( *SCHEMA_STRANDS, *MANIFEST_STRANDS, *CREDENTIAL_STRANDS, *CHILDREN_STRANDS, - *MONITOR_STRANDS, ) @@ -119,6 +112,8 @@ def _get_schema(self, strand): else: if strand not in SCHEMA_STRANDS: raise exceptions.UnknownStrand(f"Unknown strand {strand}. Try one of {ALL_STRANDS}.") + + # Get schema from twine.json file. schema_key = strand + "_schema" try: @@ -332,15 +327,7 @@ def validate_output_manifest(self, source, **kwargs): def validate_monitor_update(self, source): """Validate a monitor update against the monitors strand.""" - strand = "monitors" - data = self._load_json("monitor_update", source) - - try: - jsonschema_validate(instance=data, schema=self.monitors) - logger.debug("Validated %s against schema.", strand) - - except ValidationError as e: - raise exceptions.invalid_contents_map[strand](str(e)) + return self._validate_values(kind="monitors", source=source) @staticmethod def _get_cls(name, cls): From 8ba8bc94950ad67ca3083a9d3c59761d2e667cb1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Nov 2021 10:37:49 +0000 Subject: [PATCH 4/8] REF: Rename monitors validation method --- tests/test_monitors.py | 4 ++-- twined/twine.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_monitors.py b/tests/test_monitors.py index 254dc9b..09ce89a 100644 --- a/tests/test_monitors.py +++ b/tests/test_monitors.py @@ -22,9 +22,9 @@ def test_validate_monitors_raises_error_if_monitors_schema_not_met(self): twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) with self.assertRaises(exceptions.InvalidMonitorsUpdate): - twine.validate_monitor_update([]) + twine.validate_monitor_values([]) def test_validate_monitors_with_valid_monitor_update(self): """Test that a valid monitor update validates successfully.""" twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) - twine.validate_monitor_update({"my_property": 3.7}) + twine.validate_monitor_values({"my_property": 3.7}) diff --git a/twined/twine.py b/twined/twine.py index 10275be..74cdecf 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -325,7 +325,7 @@ def validate_output_manifest(self, source, **kwargs): """Validates the output manifest, passed as either a file or a json string""" return self._validate_manifest("output_manifest", source, **kwargs) - def validate_monitor_update(self, source): + def validate_monitor_values(self, source): """Validate a monitor update against the monitors strand.""" return self._validate_values(kind="monitors", source=source) From bde34cf3939d9d9f1d9ae47026ae6c13ca7a77dc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Nov 2021 10:39:55 +0000 Subject: [PATCH 5/8] REF: Move monitors validation method --- twined/twine.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/twined/twine.py b/twined/twine.py index 74cdecf..91f7ca5 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -313,6 +313,10 @@ def validate_output_values(self, source, **kwargs): """Validates that the output values, passed as either a file or a json string, are correct""" return self._validate_values("output_values", source, **kwargs) + def validate_monitor_values(self, source, **kwargs): + """Validate monitor values against the monitors schema strand.""" + return self._validate_values(kind="monitors", source=source, **kwargs) + def validate_configuration_manifest(self, source, **kwargs): """Validates the input manifest, passed as either a file or a json string""" return self._validate_manifest("configuration_manifest", source, **kwargs) @@ -325,10 +329,6 @@ def validate_output_manifest(self, source, **kwargs): """Validates the output manifest, passed as either a file or a json string""" return self._validate_manifest("output_manifest", source, **kwargs) - def validate_monitor_values(self, source): - """Validate a monitor update against the monitors strand.""" - return self._validate_values(kind="monitors", source=source) - @staticmethod def _get_cls(name, cls): """Getter that will return cls[name] if cls is a dict or cls otherwise""" From 733314f1eaf229a1fb1b4f135d57104906396b82 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Nov 2021 10:43:40 +0000 Subject: [PATCH 6/8] REF: Remove unnecessary new exception --- tests/test_monitors.py | 2 +- twined/exceptions.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_monitors.py b/tests/test_monitors.py index 09ce89a..980fec6 100644 --- a/tests/test_monitors.py +++ b/tests/test_monitors.py @@ -21,7 +21,7 @@ def test_validate_monitors_raises_error_if_monitors_schema_not_met(self): """Test that an error is raised if an invalid monitor update is validated.""" twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) - with self.assertRaises(exceptions.InvalidMonitorsUpdate): + with self.assertRaises(exceptions.InvalidValuesContents): twine.validate_monitor_values([]) def test_validate_monitors_with_valid_monitor_update(self): diff --git a/twined/exceptions.py b/twined/exceptions.py index 98e01a6..2ab61fc 100644 --- a/twined/exceptions.py +++ b/twined/exceptions.py @@ -105,10 +105,6 @@ class InvalidValuesContents(InvalidValues, ValidationError): """Raised when the JSON in the file is not valid according to its matching schema.""" -class InvalidMonitorsUpdate(InvalidValues, ValidationError): - pass - - # --------------------- Exceptions relating to validation of manifests ------------------------ @@ -160,6 +156,7 @@ class UnexpectedNumberOfResults(TwineException): "configuration_values": InvalidValuesJson, "input_values": InvalidValuesJson, "output_values": InvalidValuesJson, + "monitors": InvalidValuesJson, "configuration_manifest": InvalidManifestJson, "input_manifest": InvalidManifestJson, "output_manifest": InvalidManifestJson, @@ -172,8 +169,8 @@ class UnexpectedNumberOfResults(TwineException): "configuration_values": InvalidValuesContents, "input_values": InvalidValuesContents, "output_values": InvalidValuesContents, + "monitors": InvalidValuesContents, "configuration_manifest": InvalidManifestContents, "input_manifest": InvalidManifestContents, "output_manifest": InvalidManifestContents, - "monitors": InvalidMonitorsUpdate, } From 7f39a84dd9bd698c1a20a08c46636b0b61113383 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Nov 2021 17:03:48 +0000 Subject: [PATCH 7/8] REF: Rename monitors to monitor_message --- tests/test_monitors.py | 18 +++++++++--------- twined/exceptions.py | 4 ++-- twined/twine.py | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_monitors.py b/tests/test_monitors.py index 980fec6..156d3cf 100644 --- a/tests/test_monitors.py +++ b/tests/test_monitors.py @@ -2,10 +2,10 @@ from .base import BaseTestCase -class TestMonitorsTwine(BaseTestCase): - STRAND_WITH_MONITORS_SCHEMA = """ +class TestMonitorMessageTwine(BaseTestCase): + STRAND_WITH_MONITOR_MESSAGE_SCHEMA = """ { - "monitors_schema": { + "monitor_message_schema": { "type": "object", "properties": { "my_property": { @@ -17,14 +17,14 @@ class TestMonitorsTwine(BaseTestCase): } """ - def test_validate_monitors_raises_error_if_monitors_schema_not_met(self): + def test_validate_monitor_message_raises_error_if_monitor_message_schema_not_met(self): """Test that an error is raised if an invalid monitor update is validated.""" - twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) + twine = Twine(source=self.STRAND_WITH_MONITOR_MESSAGE_SCHEMA) with self.assertRaises(exceptions.InvalidValuesContents): - twine.validate_monitor_values([]) + twine.validate_monitor_message([]) - def test_validate_monitors_with_valid_monitor_update(self): + def test_validate_monitor_message_with_valid_monitor_update(self): """Test that a valid monitor update validates successfully.""" - twine = Twine(source=self.STRAND_WITH_MONITORS_SCHEMA) - twine.validate_monitor_values({"my_property": 3.7}) + twine = Twine(source=self.STRAND_WITH_MONITOR_MESSAGE_SCHEMA) + twine.validate_monitor_message({"my_property": 3.7}) diff --git a/twined/exceptions.py b/twined/exceptions.py index 2ab61fc..03a01c9 100644 --- a/twined/exceptions.py +++ b/twined/exceptions.py @@ -156,7 +156,7 @@ class UnexpectedNumberOfResults(TwineException): "configuration_values": InvalidValuesJson, "input_values": InvalidValuesJson, "output_values": InvalidValuesJson, - "monitors": InvalidValuesJson, + "monitor_message": InvalidValuesJson, "configuration_manifest": InvalidManifestJson, "input_manifest": InvalidManifestJson, "output_manifest": InvalidManifestJson, @@ -169,7 +169,7 @@ class UnexpectedNumberOfResults(TwineException): "configuration_values": InvalidValuesContents, "input_values": InvalidValuesContents, "output_values": InvalidValuesContents, - "monitors": InvalidValuesContents, + "monitor_message": InvalidValuesContents, "configuration_manifest": InvalidManifestContents, "input_manifest": InvalidManifestContents, "output_manifest": InvalidManifestContents, diff --git a/twined/twine.py b/twined/twine.py index 91f7ca5..27d53a5 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) -SCHEMA_STRANDS = ("input_values", "configuration_values", "output_values", "monitors") +SCHEMA_STRANDS = ("input_values", "configuration_values", "output_values", "monitor_message") MANIFEST_STRANDS = ( "configuration_manifest", @@ -313,9 +313,9 @@ def validate_output_values(self, source, **kwargs): """Validates that the output values, passed as either a file or a json string, are correct""" return self._validate_values("output_values", source, **kwargs) - def validate_monitor_values(self, source, **kwargs): - """Validate monitor values against the monitors schema strand.""" - return self._validate_values(kind="monitors", source=source, **kwargs) + def validate_monitor_message(self, source, **kwargs): + """Validate monitor message against the monitor message schema strand.""" + return self._validate_values(kind="monitor_message", source=source, **kwargs) def validate_configuration_manifest(self, source, **kwargs): """Validates the input manifest, passed as either a file or a json string""" From 980f35fc0b6656daea69e5d1e0f4d1844f08c80e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Nov 2021 17:09:53 +0000 Subject: [PATCH 8/8] DOC: Add example monitor message schema to docs --- docs/source/anatomy_monitors.rst | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/source/anatomy_monitors.rst b/docs/source/anatomy_monitors.rst index 8704877..7d034b3 100644 --- a/docs/source/anatomy_monitors.rst +++ b/docs/source/anatomy_monitors.rst @@ -1,13 +1,11 @@ .. _monitors_strand: -=============== -Monitors Strand -=============== +====================== +Monitor Message Strand +====================== -The ``configuration_values_schema``, ``input_values_schema`` and ``output_values_schema`` strands are *values-based*, -meaning the data that matches these strands is in JSON form. - -Each of these strands is a *json schema* which describes that data. +The ``monitor_message_schema`` strand is *values-based* meaning the data that matches the strand is in JSON form. It is +a *json schema* which describes a monitor message. .. tabs:: @@ -40,5 +38,18 @@ Let's look at basic examples for twines containing each of these strands: **Monitor data (output)** - **Log data (output)** + .. code-block:: javascript + { + "monitor_message_schema": { + "type": "object", + "properties": { + "my_property": { + "type": "number" + } + }, + "required": ["my_property"] + } + } + + **Log data (output)**