From bdd4e5b24da25b0113577373c53015b3475424b0 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 13 Dec 2019 15:46:04 +0000 Subject: [PATCH 01/20] TST Added some basic test twines --- tests/data/empty_app/twine.json | 33 ++++++++++ tests/data/example_app/input/config.json | 3 + tests/data/example_app/input/manifest.json | 21 ++++++ tests/data/example_app/twine.json | 77 ++++++++++++++++++++++ tests/data/simple_app/input/config.json | 9 +++ tests/data/simple_app/twine.json | 72 ++++++++++++++++++++ 6 files changed, 215 insertions(+) create mode 100644 tests/data/empty_app/twine.json create mode 100644 tests/data/example_app/input/config.json create mode 100644 tests/data/example_app/input/manifest.json create mode 100644 tests/data/example_app/twine.json create mode 100644 tests/data/simple_app/input/config.json create mode 100644 tests/data/simple_app/twine.json diff --git a/tests/data/empty_app/twine.json b/tests/data/empty_app/twine.json new file mode 100644 index 0000000..39b9e14 --- /dev/null +++ b/tests/data/empty_app/twine.json @@ -0,0 +1,33 @@ +{ + "children": [ + ], + "configuration_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "The example configuration form", + "description": "The configuration strand of an example twine", + "type": "object", + "properties": { + } + }, + "credentials": [ + ], + "input_manifest": [ + ], + "input_values_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Input Values", + "description": "The input values strand of an example twine", + "type": "object", + "properties": { + } + }, + "output_manifest": [ + ], + "output_values_schema": { + "title": "Output Values", + "description": "The output values strand of an example twine", + "type": "object", + "properties": { + } + } +} diff --git a/tests/data/example_app/input/config.json b/tests/data/example_app/input/config.json new file mode 100644 index 0000000..9a2c60b --- /dev/null +++ b/tests/data/example_app/input/config.json @@ -0,0 +1,3 @@ +{ + "n_iterations": 16 +} \ No newline at end of file diff --git a/tests/data/example_app/input/manifest.json b/tests/data/example_app/input/manifest.json new file mode 100644 index 0000000..c85c891 --- /dev/null +++ b/tests/data/example_app/input/manifest.json @@ -0,0 +1,21 @@ +{"id": "8ead7669-8162-4f64-8cd5-4abe92509e17", +"type": "input", +"datasets": [ + { + "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", + "tags": "met, mast, wind", + "files": [ + { + "path": "input/datasets/7ead76/file_1.csv", + "extension": "csv", + "sequence": 0 + }, + { + "path": "input/datasets/7ead76/file_2.csv", + "extension": "csv", + "sequence": 1 + } + ] + } +] +} \ No newline at end of file diff --git a/tests/data/example_app/twine.json b/tests/data/example_app/twine.json new file mode 100644 index 0000000..1895f2b --- /dev/null +++ b/tests/data/example_app/twine.json @@ -0,0 +1,77 @@ +{ + "children": [ + { + "key": "turbines", + "purpose": "wind turbines in a farm" + } + ], + "configuration_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "The example configuration form", + "description": "The configuration strand of an example twine", + "type": "object", + "properties": { + "n_iterations": { + "description": "An example of an integer configuration variable, called 'n_iterations'.", + "type": "integer", + "minimum": 1, + "maximum": 10, + "default": 5 + } + } + }, + "credentials": [ + { + "name": "MYAPI_SECRET_KEY", + "purpose": "Token for accessing the MyApi service" + }, + { + "name": "MY_DATABASE_URI", + "purpose": "A URI for accessing an external database from within a twin or analysis" + } + ], + "input_manifest": [ + { + "key": "met_mast_data", + "purpose": "A dataset containing meteorological mast data", + "filters": "tags:(met* AND mast AND location) files:(extension:csv AND sequence:>=0) location:10" + }, + { + "key": "scada_data", + "purpose": "A dataset containing scada data", + "filters": "tags:(met* AND mast) files:(extension:csv AND sequence:>=0) location:10" + } + ], + "input_values_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Input Values", + "description": "The input values strand of an example twine", + "type": "object", + "properties": { + "width": { + "description": "An example of an integer value called 'width'", + "type": "integer", + "minimum": 2 + } + } + }, + "output_manifest": [ + { + "key": "production_data", + "purpose": "A dataset containing production data", + "tags": "production, wind" + } + ], + "output_values_schema": { + "title": "Output Values", + "description": "The output values strand of an example twine", + "type": "object", + "properties": { + "width": { + "description": "An example of an integer value called 'result'", + "type": "integer", + "minimum": 2 + } + } + } +} diff --git a/tests/data/simple_app/input/config.json b/tests/data/simple_app/input/config.json new file mode 100644 index 0000000..59a6167 --- /dev/null +++ b/tests/data/simple_app/input/config.json @@ -0,0 +1,9 @@ +{ + "width": 600, + "height": 600, + "max_iterations": 16, + "color_scale": "YlGnBu", + "x_range": [-1.5, 0.6], + "y_range": [-1.26, 1.26], + "type": "png" +} \ No newline at end of file diff --git a/tests/data/simple_app/twine.json b/tests/data/simple_app/twine.json new file mode 100644 index 0000000..8b86eac --- /dev/null +++ b/tests/data/simple_app/twine.json @@ -0,0 +1,72 @@ +{ + "configuration_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Configuration for a simple app", + "description": "The app creates a mandelbrot plot", + "type": "object", + "properties": { + "width": { + "description": "Number of pixels the image contains in the x direction", + "type": "integer", + "minimum": 2, + "default": 600 + }, + "height": { + "description": "Number of pixels the image contains in the y direction", + "type": "integer", + "minimum": 2, + "default": 600 + }, + "max_iterations": { + "description": "Maximum number of iterations used to render each pixel", + "type": "integer", + "minimum": 2, + "default": 64 + }, + "color_scale": { + "description": "The colour scale string to use when mapping colours. See https://plot.ly/ipython-notebooks/color-scales/ for valid scales", + "type": "string", + "enum": ["PuBu", "YlGnBu"], + "default": "YlGnBu" + }, + "type": { + "description": "Type (jpeg or png) of the image that will be produced as a results file", + "type": "string", + "enum": ["jpg", "png"], + "default": "png" + }, + "x_range": { + "description": "The x_min to x_max range of space in which to render the fractal", + "type": "array", + "items": [{ + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + }], + "additionalItems": false, + "default": [-1.5, 0.6] + }, + "y_range": { + "description": "The y_min to y_max range of space in which to render the fractal", + "type": "array", + "items": [{ + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + }], + "additionalItems": false, + "default": [-1.26, 1.26] + } + } + }, + "output_manifest": [ + ] +} From 1e4d4c50075324ac7a2a5d7139858178a5295a55 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 13 Dec 2019 15:46:31 +0000 Subject: [PATCH 02/20] TST Added twine test harness --- tests/test_twine.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_twine.py diff --git a/tests/test_twine.py b/tests/test_twine.py new file mode 100644 index 0000000..66cf8a6 --- /dev/null +++ b/tests/test_twine.py @@ -0,0 +1,42 @@ +import unittest +from twined import Twine + + +class TestSchema(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_init_twine_with_filename(self): + """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data + """ + twine_file = 'data/simple_app/twine.json' + twine = Twine(twine_file) + + def test_empty_twine(self): + """ Ensures that an empty twine can be loaded + """ + twine_file = 'data/empty_app/twine.json' + twine = Twine(twine_file) + + def test_example_twine(self): + """ Ensures that the example (full) twine can be loaded + """ + twine_file = 'data/example_app/twine.json' + twine = Twine(twine_file) + + def test_twine_simple_configuration(self): + """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data + """ + + twine_file = 'data/simple_app/twine.json' + twine = Twine(twine_file) + + input_config_file = 'data/simple_app/input/config.json' + input_manifest_file = 'data/simple_app/input/manifest.json' + +if __name__ == '__main__': + unittest.main() From bf8c846226fa5d08982bed7a8b7d8a279f1a0cdf Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 13 Dec 2019 15:47:05 +0000 Subject: [PATCH 03/20] IMP Twine class and a range of exceptions ported across from the SDK --- twined/exceptions.py | 44 ++++++++++++++++++++++++++++++++++++++ twined/twine.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 twined/exceptions.py create mode 100644 twined/twine.py diff --git a/twined/exceptions.py b/twined/exceptions.py new file mode 100644 index 0000000..f4aa711 --- /dev/null +++ b/twined/exceptions.py @@ -0,0 +1,44 @@ + + +class TwineException(Exception): + """Any exception raised by the twine framework inherits from TwineException""" + + +class InvalidTwine(TwineException): + """Raised when the specified twine is invalid + """ + + +class InvalidInput(TwineException): + """Raise when an object is instantiated or a function called with invalid inputs + """ + + +class FolderNotPresent(InvalidInput): + """ Raise when a required folder (e.g. /input) cannot be found + """ + + +class ManifestNotFound(InvalidInput): + """ Raise when a multi manifest can not be refined to a single manifest in a search + """ + +class InvalidManifest(InvalidInput): + """Raise when a manifest loaded from JSON does not pass validation + """ + + +class InvalidManifestType(InvalidManifest): + """Raised when user attempts to create a manifest of a type other than 'input', 'output' or 'build' + """ + + +class NotImplementedYet(TwineException): + """Raised when you attempt to use a function whose high-level API is in place, but which is not implemented yet + """ + + +class UnexpectedNumberOfResults(TwineException): + """ Raise when searching for a single data file (or a particular number of data files) and the number of results exceeds that expected + """ + diff --git a/twined/twine.py b/twined/twine.py new file mode 100644 index 0000000..dc243c5 --- /dev/null +++ b/twined/twine.py @@ -0,0 +1,51 @@ +import json +import pkg_resources +from .exceptions import InvalidTwine +from jsonschema import validate, ValidationError + + +class Twine: + + def __init__(self, file=None): + self._load_twine(file) + + def _load_twine(self, file=None): + """ Load twine from a *.json file and validate its contents + """ + + # Default twine with nothing in it + if file is None: + self._raw = {} + return + + # Read the json string from the file and deserialize to objects + if not file.endswith('.json'): + raise InvalidTwine('Specified twine filename should end in ".json". Given: %s', file) + with open(file) as f: + self._raw = json.load(f) + + self._validate_twine() + + def _validate_twine(self): + """ Validate that the loaded twine contains all required parts and that each part is valid. + + A twine is itself a schema. Here we verify that the twine matches a particular schema, so this is like applying + a "schema-schema". + + """ + twine_schema = json.loads(pkg_resources.resource_string('twined', 'schema/twine_schema.json')) + + try: + validate(instance=self._raw, schema=twine_schema) + except ValidationError as e: + raise InvalidTwine(e.message) + + def validate( + self, + configuration=None, + manifest=None, + credentials=None, + monitors=None, + logs=None, + ): + pass From c28a1725ca9bb3a70556f8fe9f21f90718f66642 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 13 Dec 2019 15:47:32 +0000 Subject: [PATCH 04/20] TST Removed the placeholder test class --- tests/test_schema.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 tests/test_schema.py diff --git a/tests/test_schema.py b/tests/test_schema.py deleted file mode 100644 index ef84cfa..0000000 --- a/tests/test_schema.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest - - -class TestSchema(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass - - def test_nothing(self): - pass - - -if __name__ == '__main__': - unittest.main() From 82fe4c6a73e3e7c4ba937e323e1c054efd042e2d Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 13 Dec 2019 15:47:55 +0000 Subject: [PATCH 05/20] IMP exporting the Twine class --- twined/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/twined/__init__.py b/twined/__init__.py index e69de29..d266938 100644 --- a/twined/__init__.py +++ b/twined/__init__.py @@ -0,0 +1 @@ +from .twine import Twine From 58ad591076503a53be78b05ec08611fc3ad3fa6d Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Fri, 13 Dec 2019 15:48:32 +0000 Subject: [PATCH 06/20] DEPS version bump and added jsonschema library dependency --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8c7d9c6..ba13f1e 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,9 @@ setup( name='twined', - version='0.0.4', + version='0.0.5', py_modules=[], - install_requires=[], + install_requires=['jsonschema ~= 3.2.0'], url='https://www.github.com/octue/twined', license=license_text, author='Octue (github: octue)', From ff7c65b5b0d70e20f30f83499a6c970af2ada965 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 07:17:48 +0000 Subject: [PATCH 07/20] TST altered example manifest to a multi-dataset example --- tests/data/example_app/input/manifest.json | 64 +++++++++++++++------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/tests/data/example_app/input/manifest.json b/tests/data/example_app/input/manifest.json index c85c891..a994280 100644 --- a/tests/data/example_app/input/manifest.json +++ b/tests/data/example_app/input/manifest.json @@ -1,21 +1,43 @@ -{"id": "8ead7669-8162-4f64-8cd5-4abe92509e17", -"type": "input", -"datasets": [ - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", - "tags": "met, mast, wind", - "files": [ - { - "path": "input/datasets/7ead76/file_1.csv", - "extension": "csv", - "sequence": 0 - }, - { - "path": "input/datasets/7ead76/file_2.csv", - "extension": "csv", - "sequence": 1 - } - ] - } -] -} \ No newline at end of file +{ + "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", + "type": "input", + "datasets": [ + { + "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", + "name": "my meteorological dataset", + "tags": "met, mast, wind", + "files": [ + { + "path": "input/datasets/7ead7669/file_1.csv", + "cluster": 0, + "sequence": 0, + "extension": "csv", + "tags": "", + "posix_timestamp": null, + "data_file": { + "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + "last_modified": "2019-02-28T22:40:30.533005Z", + "name": "file_1.csv", + "size_bytes": 59684813, + "sha-512/256": "somesha" + } + }, + { + "path": "input/datasets/7ead7669/file_2.csv", + "cluster": 0, + "sequence": 1, + "extension": "csv", + "tags": "", + "posix_timestamp": null, + "data_file": { + "id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45", + "last_modified": "2019-02-28T22:40:40.633001Z", + "name": "file_2.csv", + "size_bytes": 59684813, + "sha-512/256": "someothersha" + } + } + ] + } + ] +} From a883d2f65692eb3376b6cca4583187b44e341dc9 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 07:19:03 +0000 Subject: [PATCH 08/20] TST Added tags to the example twine --- tests/data/example_app/twine.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/data/example_app/twine.json b/tests/data/example_app/twine.json index 1895f2b..394833d 100644 --- a/tests/data/example_app/twine.json +++ b/tests/data/example_app/twine.json @@ -2,7 +2,8 @@ "children": [ { "key": "turbines", - "purpose": "wind turbines in a farm" + "purpose": "wind turbines in a farm", + "filters": "tags:(met* AND mast AND location) files:(extension:csv AND sequence:>=0) location:10" } ], "configuration_schema": { From d5e918122f46a911d4858b53dbc5f078f81fc3d2 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 07:20:16 +0000 Subject: [PATCH 09/20] TST Added test class for configuration --- tests/test_twine.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_twine.py b/tests/test_twine.py index 66cf8a6..8680549 100644 --- a/tests/test_twine.py +++ b/tests/test_twine.py @@ -2,7 +2,7 @@ from twined import Twine -class TestSchema(unittest.TestCase): +class TestTwineSchema(unittest.TestCase): def setUp(self): pass @@ -35,8 +35,33 @@ def test_twine_simple_configuration(self): twine_file = 'data/simple_app/twine.json' twine = Twine(twine_file) - input_config_file = 'data/simple_app/input/config.json' - input_manifest_file = 'data/simple_app/input/manifest.json' + +class TestConfiguration(unittest.TestCase): + + def test_configuration(self): + """ + """ + twine_file = 'data/simple_app/twine.json' + twine = Twine(twine_file) + + +class TestCredentials(unittest.TestCase): + + def test_empty_credentials(self): + """ Test that a twine with no credentials will end up with empty credentials object + """ + + def test_valid_credentials_in_twine(self): + """ Test that where credentials in environment and twine match, that they import successfully and that no extra + credentials are imported + """ + + def test_exception_on_invalid_credentials(self): + """ Test that where a credential is specified in the twine that does not appear in the environment, + an exception is raised + """ + pass + if __name__ == '__main__': unittest.main() From 8924cd46110a79f49691c5067840de51875991c2 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 07:21:26 +0000 Subject: [PATCH 10/20] FEAT Added logging to the twine class --- twined/twine.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/twined/twine.py b/twined/twine.py index dc243c5..1c7465e 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -1,9 +1,13 @@ import json +import logging import pkg_resources from .exceptions import InvalidTwine from jsonschema import validate, ValidationError +logger = logging.getLogger(__name__) + + class Twine: def __init__(self, file=None): @@ -16,6 +20,7 @@ def _load_twine(self, file=None): # Default twine with nothing in it if file is None: self._raw = {} + logger.debug('No twine file specified. Loading empty twine.') return # Read the json string from the file and deserialize to objects @@ -23,20 +28,21 @@ def _load_twine(self, file=None): raise InvalidTwine('Specified twine filename should end in ".json". Given: %s', file) with open(file) as f: self._raw = json.load(f) + logger.debug('Loaded twine from file %s', file) self._validate_twine() def _validate_twine(self): """ Validate that the loaded twine contains all required parts and that each part is valid. - A twine is itself a schema. Here we verify that the twine matches a particular schema, so this is like applying - a "schema-schema". + A twine *contains* schema, but we also need to verify that it matches a certain schema itself. """ twine_schema = json.loads(pkg_resources.resource_string('twined', 'schema/twine_schema.json')) try: validate(instance=self._raw, schema=twine_schema) + logger.debug('Success: validated raw twine against schema') except ValidationError as e: raise InvalidTwine(e.message) From d842d6742cbac6fbb340f1dfd541ca4ed5c15e5e Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 07:22:07 +0000 Subject: [PATCH 11/20] DOC Added svgs for docs --- docs/source/images/digital_twin_component_for_simulation.svg | 1 + docs/source/images/digital_twin_hierarchy_extended.svg | 1 + 2 files changed, 2 insertions(+) create mode 100644 docs/source/images/digital_twin_component_for_simulation.svg create mode 100644 docs/source/images/digital_twin_hierarchy_extended.svg diff --git a/docs/source/images/digital_twin_component_for_simulation.svg b/docs/source/images/digital_twin_component_for_simulation.svg new file mode 100644 index 0000000..7d47cc6 --- /dev/null +++ b/docs/source/images/digital_twin_component_for_simulation.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/source/images/digital_twin_hierarchy_extended.svg b/docs/source/images/digital_twin_hierarchy_extended.svg new file mode 100644 index 0000000..24f415d --- /dev/null +++ b/docs/source/images/digital_twin_hierarchy_extended.svg @@ -0,0 +1 @@ + \ No newline at end of file From fb96bdadcb28ffc7e28ac0b754cd369a5ceda482 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:15:59 +0000 Subject: [PATCH 12/20] TST WIP Implementing tests for Twine class and twine validation --- ...invalid_children_dict_not_array_twine.json | 3 + .../twines/invalid_children_no_key_twine.json | 9 ++ ...alid_credentials_dict_not_array_twine.json | 6 ++ ...alid_credentials_lowercase_name_twine.json | 8 ++ .../invalid_credentials_no_name_twine.json | 7 ++ ...valid_credentials_space_in_name_twine.json | 8 ++ tests/data/twines/invalid_json_twine.json | 11 +++ tests/data/twines/valid_children_twine.json | 10 ++ tests/test_children.py | 60 ++++++++++++ tests/test_credentials.py | 62 ++++++++++++ tests/test_twine.py | 98 +++++++++++-------- 11 files changed, 241 insertions(+), 41 deletions(-) create mode 100644 tests/data/twines/invalid_children_dict_not_array_twine.json create mode 100644 tests/data/twines/invalid_children_no_key_twine.json create mode 100644 tests/data/twines/invalid_credentials_dict_not_array_twine.json create mode 100644 tests/data/twines/invalid_credentials_lowercase_name_twine.json create mode 100644 tests/data/twines/invalid_credentials_no_name_twine.json create mode 100644 tests/data/twines/invalid_credentials_space_in_name_twine.json create mode 100644 tests/data/twines/invalid_json_twine.json create mode 100644 tests/data/twines/valid_children_twine.json create mode 100644 tests/test_children.py create mode 100644 tests/test_credentials.py diff --git a/tests/data/twines/invalid_children_dict_not_array_twine.json b/tests/data/twines/invalid_children_dict_not_array_twine.json new file mode 100644 index 0000000..66245b2 --- /dev/null +++ b/tests/data/twines/invalid_children_dict_not_array_twine.json @@ -0,0 +1,3 @@ +{ + "children": {} +} diff --git a/tests/data/twines/invalid_children_no_key_twine.json b/tests/data/twines/invalid_children_no_key_twine.json new file mode 100644 index 0000000..1515fd3 --- /dev/null +++ b/tests/data/twines/invalid_children_no_key_twine.json @@ -0,0 +1,9 @@ +{ + "children": [ + { + "purpose": "Something to do with GIS data.", + "notes": "This filter (which can use the extremely powerful 'lucene' query syntax)\n allows the digital twin to locate other digital twins (public across octue or\n private in your workspace) which can provide the data you need.", + "filters": "tags:gis" + } + ] +} diff --git a/tests/data/twines/invalid_credentials_dict_not_array_twine.json b/tests/data/twines/invalid_credentials_dict_not_array_twine.json new file mode 100644 index 0000000..eb4f55a --- /dev/null +++ b/tests/data/twines/invalid_credentials_dict_not_array_twine.json @@ -0,0 +1,6 @@ +{ + "credentials": { + "name": "MY_API_SECRET_KEY", + "purpose": "Token for accessing a 3rd party API service" + } +} diff --git a/tests/data/twines/invalid_credentials_lowercase_name_twine.json b/tests/data/twines/invalid_credentials_lowercase_name_twine.json new file mode 100644 index 0000000..30afae5 --- /dev/null +++ b/tests/data/twines/invalid_credentials_lowercase_name_twine.json @@ -0,0 +1,8 @@ +{ + "credentials": [ + { + "name": "my_secrets_should_be_uppercase", + "purpose": "Token for accessing a 3rd party API service" + } + ] +} diff --git a/tests/data/twines/invalid_credentials_no_name_twine.json b/tests/data/twines/invalid_credentials_no_name_twine.json new file mode 100644 index 0000000..f809163 --- /dev/null +++ b/tests/data/twines/invalid_credentials_no_name_twine.json @@ -0,0 +1,7 @@ +{ + "credentials": [ + { + "purpose": "credentials without a name should be invalid" + } + ] +} diff --git a/tests/data/twines/invalid_credentials_space_in_name_twine.json b/tests/data/twines/invalid_credentials_space_in_name_twine.json new file mode 100644 index 0000000..522cd7c --- /dev/null +++ b/tests/data/twines/invalid_credentials_space_in_name_twine.json @@ -0,0 +1,8 @@ +{ + "credentials": [ + { + "name": "MY NAME SHOULD NOT HAVE WHITESPACE", + "purpose": "Token for accessing a 3rd party API service" + } + ] +} diff --git a/tests/data/twines/invalid_json_twine.json b/tests/data/twines/invalid_json_twine.json new file mode 100644 index 0000000..636e70e --- /dev/null +++ b/tests/data/twines/invalid_json_twine.json @@ -0,0 +1,11 @@ +{ + "children": [ + "configuration_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "The example configuration form", + "description": "The configuration strand of an example twine", + "type": "object", + "properties": { + } + }, +} diff --git a/tests/data/twines/valid_children_twine.json b/tests/data/twines/valid_children_twine.json new file mode 100644 index 0000000..2293ec6 --- /dev/null +++ b/tests/data/twines/valid_children_twine.json @@ -0,0 +1,10 @@ +{ + "children": [ + { + "key": "gis", + "purpose": "Something to do with GIS data.", + "notes": "Some internal note about how the filters work or similar.", + "filters": "tags:gis" + } + ] +} diff --git a/tests/test_children.py b/tests/test_children.py new file mode 100644 index 0000000..83ad5e7 --- /dev/null +++ b/tests/test_children.py @@ -0,0 +1,60 @@ +import unittest +from twined import Twine, exceptions + + +class TestChildrenTwine(unittest.TestCase): + """ Tests related to the twine itself - ensuring that valid and invalid + `children` entries in a twine file work as expected + """ + + def test_invalid_children_dict_not_array(self): + """ Ensures InvalidTwine exceptions are raised when instantiating twines where `children` entry is incorrectly + specified as a dict, not an array + """ + twine_file = 'data/twines/invalid_children_dict_not_array_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) + + def test_invalid_children_no_key(self): + """ Ensures InvalidTwine exceptions are raised when instantiating twines where a child + is specified without the required `key` field + """ + twine_file = 'data/twines/invalid_children_no_key_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) + + def test_valid_children(self): + """ Ensures that a twine can be instantiated with correctly specified children + """ + twine_file = 'data/twines/valid_children_twine.json' + twine = Twine(file=twine_file) + self.assertEqual(len(twine._raw['children']), 1) + + +class TestChildrenValidation(unittest.TestCase): + """ Tests related to whether validation of children occurs successfully (given a valid twine) + """ + + def test_no_children(self): + """ Test that a twine with no children will validate on an empty children input + """ + raise exceptions.NotImplementedYet() + + def test_missing_children(self): + """ Test that a twine with children will not validate on an empty children input + """ + raise exceptions.NotImplementedYet() + + def test_extra_children(self): + """ Test that a twine with no children will not validate a non-empty children input + """ + raise exceptions.NotImplementedYet() + + def test_matched_children(self): + """ Test that a twine with children required will validate when the children input matches + """ + raise exceptions.NotImplementedYet() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_credentials.py b/tests/test_credentials.py new file mode 100644 index 0000000..27931e9 --- /dev/null +++ b/tests/test_credentials.py @@ -0,0 +1,62 @@ +import unittest +from twined import Twine, exceptions + + +class TestCredentialsTwine(unittest.TestCase): + """ Tests related to the twine itself - ensuring that valid and invalid + `credentials` entries in a twine file work as expected + """ + + def test_fails_on_no_name(self): + """ Ensures InvalidTwine exceptions are raised when instantiating twines + with a missing `name` field in a credential + """ + twine_file = 'data/twines/invalid_credentials_no_name_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) + + def test_fails_on_lowercase_name(self): + """ Ensures InvalidTwine exceptions are raised when instantiating twines + with lowercase letters in the `name` field + """ + twine_file = 'data/twines/invalid_credentials_lowercase_name_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) + + def test_fails_on_dict(self): + """ Ensures InvalidTwine exceptions are raised when instantiating twines + with invalid `credentials` entries (given as a dict, not an array) + """ + twine_file = 'data/twines/invalid_credentials_dict_not_array_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) + + def test_fails_on_name_whitespace(self): + twine_file = 'data/twines/invalid_credentials_space_in_name_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) + + +class TestCredentialsValidation(unittest.TestCase): + """ Tests related to whether validation of children occurs successfully (given a valid twine) + """ + + def test_no_credentials(self): + """ Test that a twine with no credentials will validate straightforwardly + """ + raise exceptions.NotImplementedYet() + + def test_missing_credentials(self): + """ Test that a twine with credentials will not validate where they are missing from the environment + """ + raise exceptions.NotImplementedYet() + + def test_matched_credentials(self): + """ Test that a twine with credentials required will validate when the credentials are available in the + environment + """ + raise exceptions.NotImplementedYet() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_twine.py b/tests/test_twine.py index 8680549..db68749 100644 --- a/tests/test_twine.py +++ b/tests/test_twine.py @@ -1,66 +1,82 @@ import unittest -from twined import Twine +from twined import Twine, exceptions -class TestTwineSchema(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - pass +class TestTwine(unittest.TestCase): def test_init_twine_with_filename(self): - """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data + """ Ensures that the twine class can be instantiated with a file """ twine_file = 'data/simple_app/twine.json' - twine = Twine(twine_file) + Twine(file=twine_file) - def test_empty_twine(self): - """ Ensures that an empty twine can be loaded + def test_init_twine_with_json(self): + """ Ensures that a twine can be instantiated with a json string """ - twine_file = 'data/empty_app/twine.json' - twine = Twine(twine_file) + with open('data/simple_app/twine.json', 'r', encoding='utf-8') as f: + json_string = f.read() + Twine(json=json_string) - def test_example_twine(self): - """ Ensures that the example (full) twine can be loaded + def test_init_twine_with_incorrect_file_string(self): + """ Ensures that error is raised instantiating with a non-filename """ - twine_file = 'data/example_app/twine.json' - twine = Twine(twine_file) + with self.assertRaises(exceptions.MissingTwine): + Twine(file='{"mistakenly_passed": "json instead of filename"}') - def test_twine_simple_configuration(self): - """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data - """ - - twine_file = 'data/simple_app/twine.json' - twine = Twine(twine_file) - - -class TestConfiguration(unittest.TestCase): - - def test_configuration(self): + def test_init_twine_with_both_inputs(self): + """ Ensures that error is raised when attempting to instantiate with both file and json inputs """ + with self.assertRaises(exceptions.InvalidInput): + Twine( + file='data/simple_app/twine.json', + json='{"input_values": "something"}' + ) + + def test_missing_twine_file(self): + """ Ensures that an absent file raises a MissingTwine exception """ - twine_file = 'data/simple_app/twine.json' - twine = Twine(twine_file) + twine_file = 'file_is_missing.json' + with self.assertRaises(exceptions.MissingTwine): + Twine(file=twine_file) + def test_no_twine(self): + """ Tests that the canonical-but-useless case of no twine provided validates whilst issuing a warning + """ + with self.assertLogs(level='WARNING') as log: + Twine() + self.assertEqual(len(log.output), 1) + self.assertEqual(len(log.records), 1) + self.assertIn('No twine file specified', log.output[0]) -class TestCredentials(unittest.TestCase): + def test_empty_twine(self): + """ Ensures that an empty twine file can be loaded + """ + twine_file = 'data/empty_app/twine.json' + with self.assertLogs(level='DEBUG') as log: + Twine(file=twine_file) + self.assertEqual(len(log.output), 2) + self.assertEqual(len(log.records), 2) + self.assertIn('Loaded', log.output[0]) + self.assertIn('Validated', log.output[1]) - def test_empty_credentials(self): - """ Test that a twine with no credentials will end up with empty credentials object + def test_example_twine(self): + """ Ensures that the example (full) twine can be loaded and validated """ + twine_file = 'data/example_app/twine.json' + twine = Twine(file=twine_file) - def test_valid_credentials_in_twine(self): - """ Test that where credentials in environment and twine match, that they import successfully and that no extra - credentials are imported + def test_simple_twine(self): + """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data """ + twine_file = 'data/simple_app/twine.json' + twine = Twine(file=twine_file) - def test_exception_on_invalid_credentials(self): - """ Test that where a credential is specified in the twine that does not appear in the environment, - an exception is raised + def test_broken_json_twine(self): + """ Ensures that an invalid json file raises an InvalidTwine exception """ - pass + twine_file = 'data/twines/invalid_json_twine.json' + with self.assertRaises(exceptions.InvalidTwine): + Twine(file=twine_file) if __name__ == '__main__': From 75dbd511e49819de06a4c5db0689d6926f186efc Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:16:56 +0000 Subject: [PATCH 13/20] IMP Twine class can now load raw json and passes twine validation test suite --- twined/exceptions.py | 22 +++++++++------- twined/twine.py | 63 ++++++++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/twined/exceptions.py b/twined/exceptions.py index f4aa711..8f6bf11 100644 --- a/twined/exceptions.py +++ b/twined/exceptions.py @@ -1,44 +1,48 @@ class TwineException(Exception): - """Any exception raised by the twine framework inherits from TwineException""" + """ All exceptions raised by the twine framework inherits from TwineException""" class InvalidTwine(TwineException): - """Raised when the specified twine is invalid + """ Raised when the specified twine is invalid + """ + + +class MissingTwine(TwineException): + """ Raised when the specified twine file is not present """ class InvalidInput(TwineException): - """Raise when an object is instantiated or a function called with invalid inputs + """ Raised when an object is instantiated or a function called with invalid inputs """ class FolderNotPresent(InvalidInput): - """ Raise when a required folder (e.g. /input) cannot be found + """ Raised when a required folder (e.g. /input) cannot be found """ class ManifestNotFound(InvalidInput): - """ Raise when a multi manifest can not be refined to a single manifest in a search + """ Raised when a multi manifest can not be refined to a single manifest in a search """ class InvalidManifest(InvalidInput): - """Raise when a manifest loaded from JSON does not pass validation + """ Raised when a manifest loaded from JSON does not pass validation """ class InvalidManifestType(InvalidManifest): - """Raised when user attempts to create a manifest of a type other than 'input', 'output' or 'build' + """ Raised when user attempts to create a manifest of a type other than 'input', 'output' or 'build' """ class NotImplementedYet(TwineException): - """Raised when you attempt to use a function whose high-level API is in place, but which is not implemented yet + """ Raised when you attempt to use a function whose high-level API is in place, but which is not implemented yet """ class UnexpectedNumberOfResults(TwineException): """ Raise when searching for a single data file (or a particular number of data files) and the number of results exceeds that expected """ - diff --git a/twined/twine.py b/twined/twine.py index 1c7465e..1d5c752 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -1,8 +1,8 @@ -import json +import json as jsonlib import logging import pkg_resources -from .exceptions import InvalidTwine -from jsonschema import validate, ValidationError +from . import exceptions +from jsonschema import validate as jsonschema_validate, ValidationError logger = logging.getLogger(__name__) @@ -10,27 +10,49 @@ class Twine: - def __init__(self, file=None): - self._load_twine(file) + def __init__(self, **kwargs): + """ Instantiate a twine class, providing a file name or a utf-8 encoded string containing valid json. + The twine is itself validated to be correct against the twine schema. - def _load_twine(self, file=None): - """ Load twine from a *.json file and validate its contents + Note: Instantiating the twine does not validate that any inputs to an application are correct - it merely + checks that the twine itself is correct. + + """ + self._load_twine(**kwargs) + + def _load_twine(self, file=None, json=None): + """ Load twine from a *.json file or a json string and validate its contents """ # Default twine with nothing in it - if file is None: + if (file is None) and (json is None): self._raw = {} - logger.debug('No twine file specified. Loading empty twine.') + logger.warning('No twine file specified. Loading empty twine.') return - # Read the json string from the file and deserialize to objects - if not file.endswith('.json'): - raise InvalidTwine('Specified twine filename should end in ".json". Given: %s', file) - with open(file) as f: - self._raw = json.load(f) - logger.debug('Loaded twine from file %s', file) + # Decode the json string and deserialize to objects + try: + # From the file... + if file is not None: + if json is not None: + raise exceptions.InvalidInput('You cannot specify both file and json inputs') + + try: + with open(file) as f: + self._raw = jsonlib.load(f) + logger.debug('Loaded twine from file %s', file) + except FileNotFoundError as e: + raise exceptions.MissingTwine(e) + + # Directly from the string... + else: + self._raw = jsonlib.loads(json) + logger.debug('Loaded twine from input json string') - self._validate_twine() + self._validate_twine() + + except jsonlib.decoder.JSONDecodeError as e: + raise exceptions.InvalidTwine(e) def _validate_twine(self): """ Validate that the loaded twine contains all required parts and that each part is valid. @@ -38,13 +60,14 @@ def _validate_twine(self): A twine *contains* schema, but we also need to verify that it matches a certain schema itself. """ - twine_schema = json.loads(pkg_resources.resource_string('twined', 'schema/twine_schema.json')) + twine_schema = jsonlib.loads(pkg_resources.resource_string('twined', 'schema/twine_schema.json')) try: - validate(instance=self._raw, schema=twine_schema) - logger.debug('Success: validated raw twine against schema') + jsonschema_validate(instance=self._raw, schema=twine_schema) + logger.debug('Validated raw twine against schema') + except ValidationError as e: - raise InvalidTwine(e.message) + raise exceptions.InvalidTwine(e.message) def validate( self, From a633d097b34f9d91328b50af09d80d3fa8c16b9e Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:17:17 +0000 Subject: [PATCH 14/20] DOC Added developer instructions as an aide memoire --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index cc5989f..4866897 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,35 @@ A library to help digital twins talk to one another. Read more at [twined.readth [![codecov](https://codecov.io/gh/octue/twined/branch/master/graph/badge.svg)](https://codecov.io/gh/octue/twined) [![Documentation Status](https://readthedocs.org/projects/twined/badge/?version=latest)](https://twined.readthedocs.io/en/latest/?badge=latest) +## Developer notes + +### Contributing + +- Please raise an issue on the board (or add your $0.02 to an existing issue) so the maintainers know +what's happening and can advise / steer you. + +- Create a fork of twined, undertake your changes on a new branch (call it whatever you want). + +- Ask the `twined` maintainers *where* to make your pull request. We'll create a version branch, according to the +roadmap, into which you can make your PR. We'll help review the changes and improve the PR. + +- Once checks have passed, test coverage of the new code is >=95%, documentation is updated and the Review is passed, we'll merge into the version branch. + +- Once all the roadmapped features for that version are done, we'll release. + + +### Release process + +The process for creating a new release is as follows: + +1. Check out a branch for the next version, called `vX.Y.Z` +2. Create a Pull Request into the `master` branch. +3. Undertake your changes, committing and pushing to branch `vX.Y.Z` +4. Ensure that documentation is updated to match changes, and increment the changelog. **Pull requests which do not update documentation will be refused.** +5. Ensure that test coverage is sufficient. **Pull requests that decrease test coverage will be refused.** +6. Ensure code meets style guidelines (flake8 tests will fail otherwise) +7. Address Review Comments on the PR +8. Ensure the version in `setup.py` is correct and matches the branch version. +9. Merge to master. Successful test, doc build, flake8 and a new version number will automatically create the release on pypi. +10. Go to code > releases and create a new release on GitHub at the same SHA. + From 76d3a94ed7b214f57c07da2f2b86cc9300b17da2 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:19:45 +0000 Subject: [PATCH 15/20] IMP Added the main twine schema against which twines themselves are validated --- twined/schema/twine_schema.json | 109 ++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 twined/schema/twine_schema.json diff --git a/twined/schema/twine_schema.json b/twined/schema/twine_schema.json new file mode 100644 index 0000000..cf659f7 --- /dev/null +++ b/twined/schema/twine_schema.json @@ -0,0 +1,109 @@ +{ + "type": "object", + "$schema": "http://json-schema.org/2019-09/schema#", + "properties": { + "children": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "description": "A textual key identifying a group of child twins", + "type": "string" + }, + "purpose": { + "description": "What this group of child twins are used for", + "type": "string", + "default": "" + }, + "filters": { + "description": "A search term, using the Lucene Query Language, which can be used to automatically refine the list of available child twins down to ones suitable for use here.", + "type": "string", + "default": "" + } + }, + "required": [ + "key" + ] + } + }, + "configuration_schema": { + "type": "object", + "required": ["properties"] + }, + "credentials": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "description": "The credential name, in upper snake case, eg 'MYAPI_SECRET_KEY'", + "type": "string", + "pattern": "^[A-Z]+(?:_[A-Z]+)*$" + }, + "purpose": { + "description": "What this credential is used for, eg 'Token for accessing the MyApi service'", + "type": "string" + } + }, + "required": [ + "name" + ] + } + }, + "input_manifest": { + "type": "array", + "description": "A list of entries, each describing a dataset that should be attached to / made available to the digital twin", + "items": { + "type": "object", + "properties": { + "key": { + "description": "A textual key identifying this dataset within the application/twin", + "type": "string" + }, + "purpose": { + "description": "What data this dataset contains, eg 'the set of data files from the energy production calculation process'", + "type": "string", + "default": "" + }, + "filters": { + "description": "A search term, using the Lucene Query Language, which can be used to automatically refine the list of available datasets down to ones suitable for use with this twin", + "type": "string", + "default": "" + } + }, + "required": ["key"] + } + }, + "input_values_schema": { + "type": "object" + }, + "output_manifest": { + "type": "array", + "description": "A list of entries, each describing a dataset that may be created/updated when the twin is run", + "items": { + "type": "object", + "properties": { + "key": { + "description": "A textual key identifying this dataset within the application/twin", + "type": "string" + }, + "purpose": { + "description": "What data this dataset contains, eg 'the set of data files from the energy production calculation process'", + "type": "string", + "default": "" + }, + "tags": { + "description": "Comma separated tags that will be applied to the dataset when created", + "type": "string", + "default": "" + } + }, + "required": ["key"] + } + }, + "output_values_schema": { + "type": "object" + } + } +} From 2d98db7a03f73551555a7e2e44489456ef413b11 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:23:25 +0000 Subject: [PATCH 16/20] STY To flake8 --- tests/test_twine.py | 4 ++-- twined/exceptions.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_twine.py b/tests/test_twine.py index db68749..fa1beba 100644 --- a/tests/test_twine.py +++ b/tests/test_twine.py @@ -63,13 +63,13 @@ def test_example_twine(self): """ Ensures that the example (full) twine can be loaded and validated """ twine_file = 'data/example_app/twine.json' - twine = Twine(file=twine_file) + Twine(file=twine_file) def test_simple_twine(self): """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data """ twine_file = 'data/simple_app/twine.json' - twine = Twine(file=twine_file) + Twine(file=twine_file) def test_broken_json_twine(self): """ Ensures that an invalid json file raises an InvalidTwine exception diff --git a/twined/exceptions.py b/twined/exceptions.py index 8f6bf11..06361f2 100644 --- a/twined/exceptions.py +++ b/twined/exceptions.py @@ -28,6 +28,7 @@ class ManifestNotFound(InvalidInput): """ Raised when a multi manifest can not be refined to a single manifest in a search """ + class InvalidManifest(InvalidInput): """ Raised when a manifest loaded from JSON does not pass validation """ From 923ef8301432cfb67012df0e22723661a987d5f1 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:28:17 +0000 Subject: [PATCH 17/20] STY Avoided unused import error from flake8 where teh module exports --- twined/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twined/__init__.py b/twined/__init__.py index d266938..7b18929 100644 --- a/twined/__init__.py +++ b/twined/__init__.py @@ -1 +1 @@ -from .twine import Twine +from .twine import Twine # noqa: F401 From bfe0d819eb9f3f6f07b6f4891e4729cb16fc6b3e Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 12:38:40 +0000 Subject: [PATCH 18/20] DEVOPS updated test runner to capture all files --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 73449e6..f204def 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = {py36,py37},py36-flake8 setenv = PYTHONPATH = {toxinidir}:{toxinidir}/twined commands = - coverage run --source twined tests/test_schema.py + coverage run --source twined -m unittest discover coverage report --show-missing deps = jsonschema==3.0.2 From 85fd47287e81017a9b1282011f605f842980994b Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 19:26:18 +0000 Subject: [PATCH 19/20] DOC Updated docs to being WIP on a quickstart --- docs/source/examples.rst | 92 +++++++++++++++++++++++++++++++---- docs/source/index.rst | 20 +++++--- docs/source/quick_start.rst | 95 +++++++++++++++++++++++++++++++++++++ docs/source/schema.rst | 11 +++-- 4 files changed, 198 insertions(+), 20 deletions(-) create mode 100644 docs/source/quick_start.rst diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 20cb638..7bd0957 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -4,21 +4,95 @@ Examples ======== -Here, we look at example use cases for the library, and show how to use it in python. Many of these are +Here, we look at example use cases for the library, and show how to use it in python. + +It's also well worth looking at the unit test cases copied straight from the unit test cases, so you can always check there to see how everything hooks up. -.. _example_schema: +.. _example_equipment_installation_cost: -Example Schema -============== +[Simple] Equipment installation cost +==================================== .. tabs:: - .. code-tab:: py + .. group-tab:: Scenario + + You need to provide your team with an estimate for installation cost of an equipment foundation. + + It's a straightforward calculation for you, but the Logistics Team keeps changing the installation position, to + try and optimise the overall project logistics. + + Each time the locations change, the GIS team gives you an updated embedment depth, which is what you use + (along with steel cost and foundation type), to calculate cost and report it back. + + This twine allows you to define to create a wrapper around your scripts that communicates to the GIS team what you + need as an input, communicate to the logistics team what they can expect as an output. + + When deployed as a digital twin, the calculation gets automatically updated, leaving you free to get on with + all the other work! + + .. group-tab:: Twine + + We specify the ``steel_cost`` and ``foundation_type`` as ``configuration`` values, which you can set on startup of the twin. + + Once the twin is running, it requires the ``embedment_depth`` as an ``input_value`` from the GIS team. A member + of the GIS team can use your twin to get ``foundation_cost`` directly. - import numpy as np - import es + .. code-block:: javascript - def main(): - pass + { + "title": "Foundation Cost Model", + "description": "This twine helps compute the cost of an installed foundation.", + "children": [ + ], + "configuration_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Foundation cost twin configuration", + "description": "Set config parameters and constants at startup of the twin.", + "type": "object", + "properties": { + "steel_cost": { + "description": "The cost of steel in GBP/m^3. To get a better predictive model, you could add an economic twin that forecasts the cost of steel using the project timetable.", + "type": "number", + "minimum": 0, + "default": 3000 + }, + "foundation_type": { + "description": "The type of foundation being used.", + "type": "string", + "pattern": "^(monopile|twisted-jacket)$", + "default": "monopile" + } + } + }, + "input_values_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Input Values schema for the foundation cost twin", + "description": "These values are supplied to the twin asynchronously over a web socket. So as these values change, the twin can reply with an update.", + "type": "object", + "properties": { + "embedment_depth": { + "description": "Embedment depth in metres", + "type": "number", + "minimum": 10, + "maximum": 500 + } + } + }, + "output_manifest": [ + ], + "output_values_schema": { + "title": "Output Values schema for the foundation cost twin", + "description": "The response supplied to a change in input values will always conform to this schema.", + "type": "object", + "properties": { + "foundation_cost": { + "description": "The foundation cost.", + "type": "integer", + "minimum": 2 + } + } + } + } diff --git a/docs/source/index.rst b/docs/source/index.rst index 8574b0c..f35c0a4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -24,7 +24,12 @@ Twined A digital twin is a virtual representation of a real life being - a physical asset like a wind turbine or car - or even a human. Like real things, digital twins need to interact, so can be connected together, but need a common communication -framework to do so. This is what is provided by **twined**. +framework to do so. + +**twined** helps you to define a single file, a "twine", that defines a digital twin, specifying its data +interfaces, connections to other twins, and other requirements. + +Any person, or any computer, can read a twine and understand *what-goes-in* and *what-comes-out*. .. figure:: images/digital_twin_hierarchy.svg :width: 350px @@ -41,7 +46,7 @@ framework to do so. This is what is provided by **twined**. Aims ==== -**twined** provides a toolkit to help create and validate ":ref:`schema`" - descriptions of a digital twin, what data it +**twined** provides a toolkit to help create and validate "twines" - descriptions of a digital twin, what data it requires, what it does and how it works. The goals of **twined** are as follows: @@ -50,8 +55,9 @@ The goals of **twined** are as follows: - Provide functions to check that a schema itself is valid - Provide (or direct you to) tools to create schema describing what you require -Using :ref:`schema`, we can describe how digital twins connect and interact... building them together in hierarchies and -networks. +In :ref:`schema`, we describe the different parts of a twine (examining how digital twins connect and interact... +building them together in hierarchies and networks). But you may prefer to dive straight in with the :ref:`quick_start` +guide. The scope of **twined** is not large. Many other libraries will deal with hosting and deploying digital twins, still more will deal with the actual analyses done within them. **twined** purely deals with parsing and checking the @@ -107,12 +113,12 @@ and are willing to consider sponsorship of development and maintenance of that l .. toctree:: :maxdepth: 2 - :hidden: self - digital_twins - schema installation + quick_start examples + digital_twins + schema license version_history diff --git a/docs/source/quick_start.rst b/docs/source/quick_start.rst new file mode 100644 index 0000000..546ddaa --- /dev/null +++ b/docs/source/quick_start.rst @@ -0,0 +1,95 @@ +.. _quick_start: + +============ +Quick Start +============ + +.. _create_a_twine: + +Create your first twine +======================= + +Let's say we want a digital twin that accepts two values, uses them to make a calculation, then gives the result. Anyone connecting to the twin will need to know what values it requires, and what it responds with. + +First, create a blank text file, call it `twine.json`. We'll give the twin a title and description. +Paste in the following: + +.. code-block:: javascript + + { + "title": "My first digital twin... of an atomising discombobulator", + "description": "A simple example... estimates the `foz` value of an atomising discombobulator." + } + +Now, let's define an input values strand, to specify what values are required by the twin. For this we use a json schema +(you can read more about them in :ref:`introducing_json_schema`). Add the ``input_values`` field, so your twine looks like this: + +.. code-block:: javascript + + { + "title": "My first digital twin", + "description": "A simple example to build on..." + "input_values_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Input Values schema for my first digital twin", + "description": "These values are supplied to the twin by another program (often over a websocket, depending on your integration provider). So as these values change, the twin can reply with an update.", + "type": "object", + "properties": { + "foo": { + "description": "The foo value... speed of the discombobulator's input bobulation module, in m/s", + "type": "number", + "minimum": 10, + "maximum": 500 + }, + "baz": { + "description": "The baz value... period of the discombobulator's recombulation unit, in s", + "type": "number", + "minimum": 0, + "maximum": 1000 + } + } + } + } + +Finally, let's define an output values strand, to define what kind of data is returned by the twin: + +.. code-block:: javascript + + "output_values_schema": { + "$schema": "http://json-schema.org/2019-09/schema#", + "title": "Output Values schema for my first digital twin", + "description": "The twin will output data that matches this schema", + "type": "object", + "properties": { + "foz": { + "description": "Estimate of the foz value... efficiency of the discombobulator in %", + "type": "number", + "minimum": 10, + "maximum": 500 + } + } + } + + +.. _load_the_twine: + +Load the twine +============== + +**twined** provides a `Twine()` class to load a twine (from a file or a json string). +The loading process checks the twine is valid. It's as simple as: + +.. code-block:: py + + from twined import Twine + + my_twine = Twine(file='twine.json') + + +.. _validate_some_inputs: + +Validate some inputs +==================== + +.. ATTENTION:: + LIBRARY IS UNDER CONSTRUCTION! WATCH THIS SPACE! \ No newline at end of file diff --git a/docs/source/schema.rst b/docs/source/schema.rst index 7930c92..278c07e 100644 --- a/docs/source/schema.rst +++ b/docs/source/schema.rst @@ -1,10 +1,13 @@ .. _schema: -====== -Schema -====== +===================== +About Twines (Schema) +===================== -This is the core of **twined**, whose whole purpose is to provide and use schemas for digital twins.. +The core of **twined** is to provide and use schemas for digital twins. + +Below, we set out requirements and a framework for creating a *schema* to represent a digital twin. +We call these schema "twines". To just get started building a **twine**, check out the :ref:`_quick_start`. .. _requirements: From f783e0d25ec360a85e1c97c8724544cfcacd2998 Mon Sep 17 00:00:00 2001 From: Tom Clark Date: Tue, 7 Jan 2020 19:26:41 +0000 Subject: [PATCH 20/20] DOC Added developer notes for building docs locally --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 4866897..d548ba2 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ A library to help digital twins talk to one another. Read more at [twined.readth ## Developer notes +**You don't need to pay attention to this unless you plan to develop Twined.** + ### Contributing - Please raise an issue on the board (or add your $0.02 to an existing issue) so the maintainers know @@ -37,3 +39,19 @@ The process for creating a new release is as follows: 9. Merge to master. Successful test, doc build, flake8 and a new version number will automatically create the release on pypi. 10. Go to code > releases and create a new release on GitHub at the same SHA. + +### Building documents locally + +**You don't need to do this unless you plan to develop Twined.** + +- Install `doxgen`. On a mac, that's `brew install doxygen`; other systems may differ. + +- Install sphinx and other requirements for building the docs +``` +pip install -r docs/requirements.txt +``` + +- Run the build process +``` +sphinx-build -b html docs/source docs/build +``` \ No newline at end of file