diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 66a5e85..5176f2c 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -19,12 +19,13 @@ jobs: tests: runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, '#skip_ci_tests')" + if: "!contains(github.event.head_commit.message, 'skip_ci_tests')" env: USING_COVERAGE: '3.8' strategy: matrix: - python: [3.6, 3.7, 3.8] + python: [3.8] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - name: Checkout Repository uses: actions/checkout@v2 @@ -73,4 +74,3 @@ jobs: user: __token__ password: ${{ secrets.PYPI_TOKEN }} verbose: true - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..51a236b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: Publish on merge of release/x.y.z into main + +# Only trigger when a pull request into main branch is closed. +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + release: + # This job will only run if the PR has been merged (and not closed without merging). + if: github.event.pull_request.merged == true && startsWith( github.head_ref, 'release/' ) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get package version + run: echo "PACKAGE_VERSION=$(python setup.py --version)" >> $GITHUB_ENV + - name: Create Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, no need to create your own. + with: + tag_name: ${{ env.PACKAGE_VERSION }} + release_name: ${{ github.event.pull_request.title }} + body: ${{ github.event.pull_request.body }} + draft: false + prerelease: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f91bfe6..25f8fa5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,20 +13,20 @@ repos: - id: check-added-large-files args: ['--maxkb=10240'] - - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 + - repo: https://github.com/PyCQA/isort + rev: 5.7.0 hooks: - id: isort - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black args: ['--line-length', '120'] language_version: python3 - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.4 hooks: - id: flake8 language_version: python3 diff --git a/setup.py b/setup.py index 67c136d..827b119 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="twined", - version="0.0.16", + version="0.0.17", py_modules=[], install_requires=["jsonschema ~= 3.2.0", "python-dotenv"], url="https://www.github.com/octue/twined", @@ -32,10 +32,9 @@ "Intended Audience :: Developers", "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Operating System :: OS Independent", ], - python_requires=">=3.6", + python_requires=">=3.8", keywords=["digital", "twins", "data", "services", "python", "schema"], ) diff --git a/tests/base.py b/tests/base.py index 0c92a95..2df2e99 100644 --- a/tests/base.py +++ b/tests/base.py @@ -51,8 +51,8 @@ class BaseTestCase(unittest.TestCase): - """ Base test case for twined: - - sets a path to the test data directory + """Base test case for twined: + - sets a path to the test data directory """ def setUp(self): diff --git a/tests/test_children.py b/tests/test_children.py index 5578244..f2b5968 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -5,19 +5,19 @@ class TestChildrenTwine(BaseTestCase): - """ Tests related to the twine itself - ensuring that valid and invalid - `children` entries in a twine file work as expected - """ + """Tests related to the twine itself - ensuring that valid and invalid + `children` entries in a twine file work as expected + """ def test_invalid_children_dict_not_array(self): - """ Ensures InvalidTwine exceptions are raised when instantiating twines where `children` entry is incorrectly + """Ensures InvalidTwine exceptions are raised when instantiating twines where `children` entry is incorrectly specified as a dict, not an array """ with self.assertRaises(exceptions.InvalidTwine): Twine(source="""{"children": {}}""") def test_invalid_children_no_key(self): - """ Ensures InvalidTwine exceptions are raised when instantiating twines where a child + """Ensures InvalidTwine exceptions are raised when instantiating twines where a child is specified without the required `key` field """ source = """ @@ -30,8 +30,7 @@ def test_invalid_children_no_key(self): Twine(source=source) def test_valid_children(self): - """ Ensures that a twine with one child can be instantiated correctly. - """ + """Ensures that a twine with one child can be instantiated correctly.""" source = """ { "children": [{"key": "gis", "purpose": "The purpose.", "notes": "Some notes.", "filters": "tags:gis"}] @@ -40,15 +39,13 @@ def test_valid_children(self): self.assertEqual(len(Twine(source=source).children), 1) def test_empty_children(self): - """ Ensures that a twine file will validate with an empty list object as children - """ + """Ensures that a twine file will validate with an empty list object as children""" twine = Twine(source="""{"children": []}""") self.assertEqual(len(twine.children), 0) class TestChildrenValidation(BaseTestCase): - """ Tests related to whether validation of children occurs successfully (given a valid twine) - """ + """Tests related to whether validation of children occurs successfully (given a valid twine)""" VALID_TWINE_WITH_CHILDREN = """ { @@ -71,19 +68,16 @@ class TestChildrenValidation(BaseTestCase): """ def test_no_children(self): - """ Test that a twine with no children will validate on an empty children input - """ + """Test that a twine with no children will validate on an empty children input""" Twine().validate_children(source=[]) def test_missing_children(self): - """ Test that a twine with children will not validate on an empty children input - """ + """Test that a twine with children will not validate on an empty children input""" with self.assertRaises(exceptions.InvalidValuesContents): Twine(source=self.VALID_TWINE_WITH_CHILDREN).validate_children(source=[]) def test_extra_children(self): - """ Test that a twine with no children will not validate a non-empty children input - """ + """Test that a twine with no children will not validate a non-empty children input""" with self.assertRaises(exceptions.InvalidValuesContents): Twine().validate_children(source=self.VALID_CHILD_VALUE) @@ -95,8 +89,7 @@ def test_backend_cannot_be_empty(self): Twine().validate_children(source=single_child_missing_backend) def test_extra_key_validation_on_empty_twine(self): - """ Test that children with extra data will not raise a validation error on an empty twine. - """ + """Test that children with extra data will not raise a validation error on an empty twine.""" children_values_with_extra_data = """ [ {"key": "gis", "id": "id", "uri_env_name": "VAR_NAME", "an_extra_key": "not a problem if present"}, @@ -108,7 +101,7 @@ def test_extra_key_validation_on_empty_twine(self): Twine().validate_children(source=children_values_with_extra_data) def test_extra_key_validation_on_valid_twine(self): - """ Test that children with extra data will not raise a validation error on a non-empty valid twine. + """Test that children with extra data will not raise a validation error on a non-empty valid twine. # TODO review this behaviour - possibly should raise an error but allow for a user specified extra_data property """ single_child_with_extra_data = """ @@ -130,8 +123,7 @@ def test_extra_key_validation_on_valid_twine(self): twine.validate_children(source=single_child_with_extra_data) def test_invalid_env_name(self): - """ Test that a child uri env name not in ALL_CAPS_SNAKE_CASE doesn't validate - """ + """Test that a child uri env name not in ALL_CAPS_SNAKE_CASE doesn't validate""" child_with_invalid_environment_variable_name = """ [ { @@ -146,13 +138,12 @@ def test_invalid_env_name(self): Twine().validate_children(source=child_with_invalid_environment_variable_name) def test_invalid_json(self): - """ Tests that a children entry with invalid json will raise an error - """ + """Tests that a children entry with invalid json will raise an error""" with self.assertRaises(exceptions.InvalidValuesJson): Twine(source=self.VALID_TWINE_WITH_CHILDREN).validate_children(source="[") def test_valid(self): - """ Test that a valid twine will validate valid children + """Test that a valid twine will validate valid children Valiantly and Validly validating validity since 1983. To those reading this, know that YOU'RE valid. """ diff --git a/tests/test_credentials.py b/tests/test_credentials.py index 5c6ff98..34253d7 100644 --- a/tests/test_credentials.py +++ b/tests/test_credentials.py @@ -7,12 +7,12 @@ class TestCredentialsTwine(BaseTestCase): - """ Tests related to the twine itself - ensuring that valid and invalid `credentials` entries in a twine file work + """Tests related to the twine itself - ensuring that valid and invalid `credentials` entries in a twine file work as expected. """ def test_fails_on_no_name(self): - """ Ensures InvalidTwine exceptions are raised when instantiating twines with a missing `name` field in a + """Ensures InvalidTwine exceptions are raised when instantiating twines with a missing `name` field in a credential. """ invalid_credentials_no_name_twine = """ @@ -29,7 +29,7 @@ def test_fails_on_no_name(self): Twine(source=invalid_credentials_no_name_twine) def test_fails_on_lowercase_name(self): - """ Ensures InvalidTwine exceptions are raised when instantiating twines with lowercase letters in the `name` + """Ensures InvalidTwine exceptions are raised when instantiating twines with lowercase letters in the `name` field. """ invalid_credentials_lowercase_name_twine = """ @@ -47,7 +47,7 @@ def test_fails_on_lowercase_name(self): Twine(source=invalid_credentials_lowercase_name_twine) def test_fails_on_dict(self): - """ Ensures InvalidTwine exceptions are raised when instantiating twines with invalid `credentials` entries + """Ensures InvalidTwine exceptions are raised when instantiating twines with invalid `credentials` entries (given as a dict, not an array). """ invalid_credentials_dict_not_array_twine = """ @@ -79,8 +79,7 @@ def test_fails_on_name_whitespace(self): class TestCredentialsValidation(BaseTestCase): - """ Tests related to whether validation of children occurs successfully (given a valid twine) - """ + """Tests related to whether validation of children occurs successfully (given a valid twine)""" VALID_CREDENTIALS_TWINE = """ { @@ -94,50 +93,32 @@ class TestCredentialsValidation(BaseTestCase): "purpose": "Token for accessing a 3rd party API service" }, { - "name": "SECRET_THE_THIRD", - "purpose": "Usually a big secret but sometimes has a convenient non-secret default, like a sandbox or local database", - "default": "postgres://pguser:pgpassword@localhost:5432/pgdb" + "name": "SECRET_THE_THIRD" } ] } """ def test_no_credentials(self): - """ Test that a twine with no credentials will validate straightforwardly - """ + """Test that a twine with no credentials will validate straightforwardly""" twine = Twine(source=VALID_SCHEMA_TWINE) twine.validate_credentials() def test_missing_credentials(self): - """ Test that a twine with credentials will not validate where they are missing from the environment - """ + """Test that a twine with credentials will not validate where they are missing from the environment""" twine = Twine(source=self.VALID_CREDENTIALS_TWINE) with self.assertRaises(exceptions.CredentialNotFound): twine.validate_credentials() - def test_default_credentials(self): - """ Test that a twine with credentials will validate where ones with defaults are missing from the environment - """ - twine = Twine(source=self.VALID_CREDENTIALS_TWINE) - with mock.patch.dict(os.environ, {"SECRET_THE_FIRST": "a value", "SECRET_THE_SECOND": "another value"}): - credentials = twine.validate_credentials() - - self.assertIn("SECRET_THE_FIRST", credentials.keys()) - self.assertIn("SECRET_THE_SECOND", credentials.keys()) - self.assertIn("SECRET_THE_THIRD", credentials.keys()) - self.assertEqual(credentials["SECRET_THE_THIRD"], "postgres://pguser:pgpassword@localhost:5432/pgdb") - - def test_nondefault_credentials(self): - """ Test that the environment will override a default value for a credential - """ + def test_credentials(self): + """ Test that the environment will override a default value for a credential.""" twine = Twine(source=self.VALID_CREDENTIALS_TWINE) with mock.patch.dict( os.environ, - {"SECRET_THE_FIRST": "a value", "SECRET_THE_SECOND": "another value", "SECRET_THE_THIRD": "nondefault"}, + {"SECRET_THE_FIRST": "a value", "SECRET_THE_SECOND": "another value", "SECRET_THE_THIRD": "value"}, ): - credentials = twine.validate_credentials() - - self.assertEqual(credentials["SECRET_THE_THIRD"], "nondefault") + twine.validate_credentials() + self.assertEqual(os.environ["SECRET_THE_THIRD"], "value") if __name__ == "__main__": diff --git a/tests/test_manifest_strands.py b/tests/test_manifest_strands.py index 8117ccb..8da4190 100644 --- a/tests/test_manifest_strands.py +++ b/tests/test_manifest_strands.py @@ -6,8 +6,7 @@ class TestManifestStrands(BaseTestCase): - """ Testing operation of the Twine class for validation of data using strands which require manifests - """ + """Testing operation of the Twine class for validation of data using strands which require manifests""" VALID_MANIFEST_STRAND = """ { @@ -41,8 +40,7 @@ class TestManifestStrands(BaseTestCase): """ def test_missing_manifest_files(self): - """ Ensures that if you try to read values from missing files, the right exceptions get raised - """ + """Ensures that if you try to read values from missing files, the right exceptions get raised""" twine = Twine(source=self.VALID_MANIFEST_STRAND) file = os.path.join(self.path, "not_a_file.json") @@ -56,8 +54,7 @@ def test_missing_manifest_files(self): twine.validate_output_manifest(source=file) def test_valid_manifest_files(self): - """ Ensures that a manifest file will validate - """ + """Ensures that a manifest file will validate""" valid_configuration_manifest = """ { "id": "3ead7669-8162-4f64-8cd5-4abe92509e17", diff --git a/tests/test_schema_strands.py b/tests/test_schema_strands.py index b061011..d9da440 100644 --- a/tests/test_schema_strands.py +++ b/tests/test_schema_strands.py @@ -7,13 +7,12 @@ class TestSchemaStrands(BaseTestCase): - """ Testing operation of the Twine class for validation of data using strands which contain schema - """ + """Testing operation of the Twine class for validation of data using strands which contain schema""" VALID_CONFIGURATION_VALUE = """{"n_iterations": 1}""" def test_invalid_strand(self): - """ Ensures that an incorrect strand name would lead to the correct exception + """Ensures that an incorrect strand name would lead to the correct exception Note: This tests an internal method. The current API doesn't allow this error to emerge but tthis check allows us to extend to a generic method """ @@ -23,8 +22,7 @@ def test_invalid_strand(self): twine._validate_against_schema("not_a_strand_name", data) def test_missing_values_files(self): - """ Ensures that if you try to read values from missing files, the right exceptions get raised - """ + """Ensures that if you try to read values from missing files, the right exceptions get raised""" twine = Twine(source=VALID_SCHEMA_TWINE) values_file = os.path.join(self.path, "not_a_file.json") @@ -38,20 +36,17 @@ def test_missing_values_files(self): twine.validate_output_values(source=values_file) def test_no_values(self): - """ Ensures that giving no data source raises an invalidJson error - """ + """Ensures that giving no data source raises an invalidJson error""" with self.assertRaises(exceptions.InvalidValuesJson): Twine(source=VALID_SCHEMA_TWINE).validate_configuration_values(source=None) def test_empty_values(self): - """ Ensures that appropriate errors are generated for invalid values - """ + """Ensures that appropriate errors are generated for invalid values""" with self.assertRaises(exceptions.InvalidValuesJson): Twine(source=VALID_SCHEMA_TWINE).validate_configuration_values(source="") def test_strand_not_found(self): - """ Ensures that if a twine doesn't have a strand, you can't validate against it - """ + """Ensures that if a twine doesn't have a strand, you can't validate against it""" valid_no_output_schema_twine = """ { "configuration_values_schema": { @@ -76,26 +71,22 @@ def test_strand_not_found(self): Twine(source=valid_no_output_schema_twine).validate_output_values(source="{}") def test_incorrect_values(self): - """ Ensures that appropriate errors are generated for invalid values - """ + """Ensures that appropriate errors are generated for invalid values""" incorrect_configuration_value = """{"n_iterations": "should not be a string, this field requires an integer"}""" with self.assertRaises(exceptions.InvalidValuesContents): Twine(source=VALID_SCHEMA_TWINE).validate_configuration_values(source=incorrect_configuration_value) def test_missing_not_required_values(self): - """ Ensures that appropriate errors are generated for missing values - """ + """Ensures that appropriate errors are generated for missing values""" Twine(source=VALID_SCHEMA_TWINE).validate_output_values(source="{}") def test_missing_required_values(self): - """ Ensures that appropriate errors are generated for missing values - """ + """Ensures that appropriate errors are generated for missing values""" with self.assertRaises(exceptions.InvalidValuesContents): Twine(source=VALID_SCHEMA_TWINE).validate_input_values(source="{}") def test_valid_values_files(self): - """ Ensures that values can be read and validated correctly from files on disk - """ + """Ensures that values can be read and validated correctly from files on disk""" twine = Twine(source=VALID_SCHEMA_TWINE) with TemporaryDirectory() as tmp_dir: @@ -105,13 +96,11 @@ def test_valid_values_files(self): twine.validate_output_values(source="""{"width": 36}""") def test_valid_values_json(self): - """ Ensures that values can be read and validated correctly from a json string - """ + """Ensures that values can be read and validated correctly from a json string""" Twine(source=VALID_SCHEMA_TWINE).validate_configuration_values(source=self.VALID_CONFIGURATION_VALUE) def test_valid_with_extra_values(self): - """ Ensures that extra values get ignored - """ + """Ensures that extra values get ignored""" configuration_valid_with_extra_field = """ { "n_iterations": 1, diff --git a/tests/test_twine.py b/tests/test_twine.py index f73b187..9e27977 100644 --- a/tests/test_twine.py +++ b/tests/test_twine.py @@ -6,35 +6,29 @@ class TestTwine(BaseTestCase): - """ Testing operation of the Twine class - """ + """Testing operation of the Twine class""" def test_init_twine_with_filename(self): - """ Ensures that the twine class can be instantiated with a file - """ + """Ensures that the twine class can be instantiated with a file""" Twine(source=os.path.join(self.path, "apps", "simple_app", "twine.json")) def test_init_twine_with_json(self): - """ Ensures that a twine can be instantiated with a json string - """ + """Ensures that a twine can be instantiated with a json string""" with open(os.path.join(self.path, "apps", "simple_app", "twine.json"), "r", encoding="utf-8") as f: Twine(source=f.read()) def test_no_twine(self): - """ Tests that the canonical-but-useless case of no twine provided validates empty - """ + """Tests that the canonical-but-useless case of no twine provided validates empty""" Twine() def test_incorrect_version_twine(self): - """ Ensures exception is thrown on mismatch between installed and specified versions of twined - """ + """Ensures exception is thrown on mismatch between installed and specified versions of twined""" incorrect_version_twine = """{"twined_version": "0.0.0"}""" with self.assertRaises(exceptions.TwineVersionConflict): Twine(source=incorrect_version_twine) def test_empty_twine(self): - """ Ensures that an empty twine file can be loaded - """ + """Ensures that an empty twine file can be loaded""" with self.assertLogs(level="DEBUG") as log: Twine(source="{}") self.assertEqual(len(log.output), 3) @@ -43,18 +37,15 @@ def test_empty_twine(self): self.assertIn("Validated", log.output[1]) def test_example_twine(self): - """ Ensures that the example (full) twine can be loaded and validated - """ + """Ensures that the example (full) twine can be loaded and validated""" Twine(source=os.path.join(self.path, "apps", "example_app", "twine.json")) def test_simple_twine(self): - """ Ensures that the simple app schema can be loaded and used to parse some basic config and values data - """ + """Ensures that the simple app schema can be loaded and used to parse some basic config and values data""" Twine(source=os.path.join(self.path, "apps", "simple_app", "twine.json")) def test_broken_json_twine(self): - """ Ensures that an invalid json file raises an InvalidTwine exception - """ + """Ensures that an invalid json file raises an InvalidTwine exception""" invalid_json_twine = """ { "children": [ diff --git a/tests/test_utils.py b/tests/test_utils.py index de3bb53..97b1ef2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,12 +10,10 @@ class TestUtils(BaseTestCase): - """ Testing operation of the Twine class - """ + """Testing operation of the Twine class""" def test_load_json_with_file_like(self): - """ Ensures that json can be loaded from a file-like object - """ + """Ensures that json can be loaded from a file-like object""" with TemporaryDirectory() as tmp_dir: with open(self._write_json_string_to_file(VALID_SCHEMA_TWINE, tmp_dir), "r") as file_like: data = load_json(file_like) @@ -23,23 +21,20 @@ def test_load_json_with_file_like(self): self.assertIn(key, ("configuration_values_schema", "input_values_schema", "output_values_schema")) def test_load_json_with_object(self): - """ Ensures if load_json is called on an already loaded object, it'll pass-through successfully - """ + """Ensures if load_json is called on an already loaded object, it'll pass-through successfully""" already_loaded_data = {"a": 1, "b": 2} data = load_json(already_loaded_data) for key in data.keys(): self.assertIn(key, ("a", "b")) def test_load_json_with_disallowed_kind(self): - """ Ensures that when attempting to load json with a kind which is diallowed, the correct exception is raised - """ + """Ensures that when attempting to load json with a kind which is diallowed, the correct exception is raised""" custom_allowed_kinds = ("file-like", "filename", "object") # Removed "string" with self.assertRaises(exceptions.InvalidSourceKindException): load_json("{}", allowed_kinds=custom_allowed_kinds) def test_encoder_without_numpy(self): - """ Ensures that the json encoder can work without numpy being installed - """ + """Ensures that the json encoder can work without numpy being installed""" some_json = {"a": np.array([0, 1])} with mock.patch("twined.utils.encoders._numpy_spec", new=None): with self.assertRaises(TypeError) as e: @@ -51,8 +46,7 @@ def test_encoder_without_numpy(self): self.assertTrue(py36 or py38) def test_encoder_with_numpy(self): - """ Ensures that the json encoder can work with numpy installed - """ + """Ensures that the json encoder can work with numpy installed""" some_json = {"a": np.array([0, 1])} json.dumps(some_json, cls=TwinedEncoder) diff --git a/tox.ini b/tox.ini index 85a1cce..57f36f6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py36,py37,py38},py36-flake8 +envlist = {py38} [testenv] setenv = @@ -9,9 +9,3 @@ commands = coverage report --show-missing coverage xml deps = -r requirements-dev.txt - -[testenv:py36-flake8] -commands = flake8 . -deps = - flake8 - flake8-print diff --git a/twined/exceptions.py b/twined/exceptions.py index 81379f9..959556d 100644 --- a/twined/exceptions.py +++ b/twined/exceptions.py @@ -2,153 +2,126 @@ class TwineException(Exception): - """ All exceptions raised by the twine framework inherit from TwineException - """ + """All exceptions raised by the twine framework inherit from TwineException""" class NotImplementedYet(TwineException): - """ Raised when you attempt to use a function whose high-level API is in place, but which is not implemented yet - """ + """Raised when you attempt to use a function whose high-level API is in place, but which is not implemented yet""" class TwineValueException(TwineException, ValueError): - """ Raised when a python ValueError is appropriate to ensure all errors still also inherit from TwineException - """ + """Raised when a python ValueError is appropriate to ensure all errors still also inherit from TwineException""" class TwineTypeException(TwineException, TypeError): - """ Raised when a python TypeError is appropriate to ensure all errors still also inherit from TwineException - """ + """Raised when a python TypeError is appropriate to ensure all errors still also inherit from TwineException""" class TwineVersionConflict(TwineException): - """ Raised when the (optional) "twined_version" field in the twine file does not match the current installed version of twined - """ + """Raised when the (optional) "twined_version" field in the twine file does not match the current installed version of twined""" # --------------------- Exceptions relating to the twine itself ------------------------ class InvalidTwine(TwineException): - """ Raised when the specified twine is invalid for some reason - """ + """Raised when the specified twine is invalid for some reason""" class InvalidTwineJson(InvalidTwine): - """ Raised when the JSON in the twine file is broken - """ + """Raised when the JSON in the twine file is broken""" class InvalidTwineContents(InvalidTwine, ValidationError): - """ Raised when the JSON in the twine file is not valid (eg doesn't match twine schema) - """ + """Raised when the JSON in the twine file is not valid (eg doesn't match twine schema)""" # --------------------- Exceptions relating to accessing/setting strands ------------------------ class UnknownStrand(TwineException, ValueError): - """ Raised when referencing a strand which is not defined in ALL_STRANDS - """ + """Raised when referencing a strand which is not defined in ALL_STRANDS""" class StrandNotFound(TwineException, KeyError): - """ Raised when the attempting to access a strand not present in the twine - """ + """Raised when the attempting to access a strand not present in the twine""" # --------------------- Exceptions relating to missing files/folders ------------------------ class FolderNotFound(TwineException): - """ Raised when a required folder (e.g. /input) cannot be found - """ + """Raised when a required folder (e.g. /input) cannot be found""" class CredentialNotFound(TwineException): - """ Raised when a credential specified in the twine file is not present in either the environment or a .env file - """ + """Raised when a credential specified in the twine file is not present in either the environment or a .env file""" class TwineFileNotFound(TwineException, FileNotFoundError): - """ Raised when the specified twine file is not present - """ + """Raised when the specified twine file is not present""" class ConfigurationValuesFileNotFound(TwineException, FileNotFoundError): - """ Raised when attempting to read configuration values from a file that is missing - """ + """Raised when attempting to read configuration values from a file that is missing""" class ConfigurationManifestFileNotFound(TwineException, FileNotFoundError): - """ Raised when a configuration manifest file is required by a twine, but is not present in the input directory - """ + """Raised when a configuration manifest file is required by a twine, but is not present in the input directory""" class InputManifestFileNotFound(TwineException, FileNotFoundError): - """ Raised when an input manifest file is required by a twine, but is not present in the input directory - """ + """Raised when an input manifest file is required by a twine, but is not present in the input directory""" class InputValuesFileNotFound(TwineException, FileNotFoundError): - """ Raised when attempting to read input values from a file that is missing - """ + """Raised when attempting to read input values from a file that is missing""" class OutputManifestFileNotFound(TwineException, FileNotFoundError): - """ Raised when twined checks that output manifest file has been produced, but it is not present in the output directory - """ + """Raised when twined checks that output manifest file has been produced, but it is not present in the output directory""" class OutputValuesFileNotFound(TwineException, FileNotFoundError): - """ Raised when attempting to read output values from a file that is missing - """ + """Raised when attempting to read output values from a file that is missing""" # --------------------- Exceptions relating to validation of JSON data (input, output, config values) ------------------ class InvalidSourceKindException(TwineException): - """ Raised when attempting to use the json loader for a disallowed kind - """ + """Raised when attempting to use the json loader for a disallowed kind""" class InvalidValues(TwineException): - """ Raised when JSON data (like Config data, Input Values or Output Values) is invalid - """ + """Raised when JSON data (like Config data, Input Values or Output Values) is invalid""" class InvalidValuesJson(InvalidValues): - """ Raised when the JSON in the file or string is broken so cannot be parsed - """ + """Raised when the JSON in the file or string is broken so cannot be parsed""" class InvalidValuesContents(InvalidValues, ValidationError): - """ Raised when the JSON in the file is not valid according to its matching schema. - """ + """Raised when the JSON in the file is not valid according to its matching schema.""" # --------------------- Exceptions relating to validation of manifests ------------------------ class InvalidManifest(TwineException): - """ Raised when a manifest loaded from JSON does not pass validation - """ + """Raised when a manifest loaded from JSON does not pass validation""" class InvalidManifestJson(InvalidManifest): - """ Raised when the json in the manifest file is broken - """ + """Raised when the json in the manifest file is broken""" class InvalidManifestType(InvalidManifest): - """ Raised when user attempts to create a manifest of an invalid type - """ + """Raised when user attempts to create a manifest of an invalid type""" class InvalidManifestContents(InvalidManifest, ValidationError): - """ Raised when the manifest files are missing or do not match tags, sequences, clusters, extensions etc as required - """ + """Raised when the manifest files are missing or do not match tags, sequences, clusters, extensions etc as required""" # --------------------- Exceptions relating to access of data using the Twine instance ------------------------ @@ -156,8 +129,7 @@ class InvalidManifestContents(InvalidManifest, ValidationError): # TODO This is related to filtering files from a manifest. Determine whether this belongs here, # or whether we should port the filtering code across from the SDK. class UnexpectedNumberOfResults(TwineException): - """ Raise when searching for a single data file (or a particular number of data files) and the number of results exceeds that expected - """ + """Raise when searching for a single data file (or a particular number of data files) and the number of results exceeds that expected""" # --------------------- Maps allowing customised exceptions per-strand (simplifies code elsewhere) ------------------ diff --git a/twined/schema/twine_schema.json b/twined/schema/twine_schema.json index cf659f7..9522ffb 100644 --- a/twined/schema/twine_schema.json +++ b/twined/schema/twine_schema.json @@ -48,7 +48,8 @@ }, "required": [ "name" - ] + ], + "additionalProperties": false } }, "input_manifest": { diff --git a/twined/twine.py b/twined/twine.py index 2ac0d88..2a4e18f 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -40,7 +40,7 @@ class Twine: - """ Twine class manages validation of inputs and outputs to/from a data service, based on spec in a 'twine' file. + """Twine class manages validation of inputs and outputs to/from a data service, based on spec in a 'twine' file. Instantiate a Twine by providing a file name or a utf-8 encoded string containing valid json. The twine is itself validated to be correct on instantiation of Twine(). @@ -51,16 +51,14 @@ class Twine: """ def __init__(self, **kwargs): - """ Constructor for the twine class - """ + """Constructor for the twine class""" for name, strand in self._load_twine(**kwargs).items(): setattr(self, name, strand) self._available_strands = tuple(trim_suffix(name, "_schema") for name in vars(self)) def _load_twine(self, source=None): - """ Load twine from a *.json filename, file-like or a json string and validates twine contents - """ + """Load twine from a *.json filename, file-like or a json string and validates twine contents""" if source is None: # If loading an unspecified twine, return an empty one rather than raising error (like in _load_data()) @@ -74,13 +72,12 @@ def _load_twine(self, source=None): return raw def _load_json(self, kind, source, **kwargs): - """ Loads data from either a *.json file, an open file pointer or a json string. Directly returns any other data - """ + """Loads data from either a *.json file, an open file pointer or a json string. Directly returns any other data""" if source is None: - raise exceptions.invalid_json_map[kind](f"Cannot load {kind} - no data source specified") + raise exceptions.invalid_json_map[kind](f"Cannot load {kind} - no data source specified.") - # Decode the json string and deserialize to objects + # Decode the json string and deserialize to objects. try: data = load_json(source, **kwargs) except FileNotFoundError as e: @@ -92,7 +89,7 @@ def _load_json(self, kind, source, **kwargs): return data def _validate_against_schema(self, strand, data): - """ Validates data against a schema, raises exceptions of type InvalidJson if not compliant. + """Validates data against a schema, raises exceptions of type InvalidJson if not compliant. Can be used to validate: - the twine file contents itself against the present version twine spec @@ -113,7 +110,7 @@ def _validate_against_schema(self, strand, data): elif strand in MANIFEST_STRANDS: # The data is a manifest of files. The "*_manifest" strands of the twine describe matching criteria used to # filter files appropriate for consumption by the digital twin, not the schema of the manifest data, which - # is distributed with thie package to ensure version consistency... + # is distributed with this package to ensure version consistency... schema = jsonlib.loads(pkg_resources.resource_string("twined", "schema/manifest_schema.json")) else: @@ -133,8 +130,7 @@ def _validate_against_schema(self, strand, data): raise exceptions.invalid_contents_map[strand](str(e)) def _validate_twine_version(self, twine_file_twined_version): - """ Validates that the installed version is consistent with an optional version specification in the twine file - """ + """Validates that the installed version is consistent with an optional version specification in the twine file""" installed_twined_version = pkg_resources.get_distribution("twined").version logger.debug( "Twine versions... %s installed, %s specified in twine", installed_twined_version, twine_file_twined_version @@ -145,8 +141,7 @@ def _validate_twine_version(self, twine_file_twined_version): ) def _validate_values(self, kind, source, cls=None, **kwargs): - """ Common values validator method - """ + """Common values validator method""" data = self._load_json(kind, source, **kwargs) self._validate_against_schema(kind, data) if cls: @@ -154,8 +149,7 @@ def _validate_values(self, kind, source, cls=None, **kwargs): return data def _validate_manifest(self, kind, source, cls=None, **kwargs): - """ Common manifest validator method - """ + """Common manifest validator method""" data = self._load_json(kind, source, **kwargs) # TODO elegant way of cleaning up this nasty serialisation hack to manage conversion of outbound manifests to primitive @@ -174,13 +168,11 @@ def _validate_manifest(self, kind, source, cls=None, **kwargs): @property def available_strands(self): - """ Tuple of strand names that are found in this twine - """ + """Tuple of strand names that are found in this twine""" return self._available_strands def validate_children(self, source, **kwargs): - """ Validates that the children values, passed as either a file or a json string, are correct - """ + """Validates that the children values, passed as either a file or a json string, are correct""" # TODO cache this loaded data keyed on a hashed version of kwargs children = self._load_json("children", source, **kwargs) self._validate_against_schema("children", children) @@ -215,16 +207,16 @@ def validate_children(self, source, **kwargs): # TODO Additional validation that the children match what is set as required in the Twine return children - def validate_credentials(self, dotenv_path=None): - """ Validates that all credentials required by the twine are present + def validate_credentials(self, *args, dotenv_path=None, **kwargs): + """Validate that all credentials required by the twine are present. - Credentials may either be set as environment variables or defined in a '.env' file. If not present in the - environment, validate_credentials will check for variables in a .env file (if present) and populate the - environment with them. If not present in either the environment or the .env file, default values are used - (if defined) or an error is thrown. + Credentials must be set as environment variables, or defined in a '.env' file. If stored remotely in a secrets + manager (e.g. Google Cloud Secrets), they must be loaded into the environment before validating the credentials + strand. - Typically a .env file resides at the root of your application (the working directory) although a specific path - may be set using the `dotenv_path` argument. + If not present in the environment, validate_credentials will check for variables in a .env file (if present) + and populate the environment with them. Typically a .env file resides at the root of your application (the + working directory) although a specific path may be set using the `dotenv_path` argument. .env files should never be committed to git or any other version control system. @@ -242,60 +234,52 @@ def validate_credentials(self, dotenv_path=None): export MULTILINE_VAR="hello\nworld" ``` """ + if not hasattr(self, "credentials"): + return set() - # Load any variables from the .env file into the environment + # Load any variables from the .env file into the environment. dotenv_path = dotenv_path or os.path.join(".", ".env") load_dotenv(dotenv_path) - # Loop through the required credentials to check for presence of each - credentials = {} - for credential in getattr(self, "credentials", []): - name = credential["name"] - default = credential.get("default", None) - credentials[name] = os.environ.get(name, default) - if credentials[name] is None: - raise exceptions.CredentialNotFound(f"Credential '{name}' missing from environment or .env file") + for credential in self.credentials: + if credential["name"] not in os.environ: + raise exceptions.CredentialNotFound( + f"Credential {credential['name']!r} missing from environment or .env file." + ) - return credentials + return self.credentials def validate_configuration_values(self, source, **kwargs): - """ Validates that the configuration values, passed as either a file or a json string, are correct - """ + """Validates that the configuration values, passed as either a file or a json string, are correct""" return self._validate_values("configuration_values", source, **kwargs) def validate_input_values(self, source, **kwargs): - """ Validates that the input values, passed as either a file or a json string, are correct - """ + """Validates that the input values, passed as either a file or a json string, are correct""" return self._validate_values("input_values", source, **kwargs) def validate_output_values(self, source, **kwargs): - """ Validates that the output values, passed as either a file or a json string, are correct - """ + """Validates that the output values, passed as either a file or a json string, are correct""" return self._validate_values("output_values", source, **kwargs) def validate_configuration_manifest(self, source, **kwargs): - """ Validates the input manifest, passed as either a file or a json string - """ + """Validates the input manifest, passed as either a file or a json string""" return self._validate_manifest("configuration_manifest", source, **kwargs) def validate_input_manifest(self, source, **kwargs): - """ Validates the input manifest, passed as either a file or a json string - """ + """Validates the input manifest, passed as either a file or a json string""" return self._validate_manifest("input_manifest", source, **kwargs) def validate_output_manifest(self, source, **kwargs): - """ Validates the output manifest, passed as either a file or a json string - """ + """Validates the output manifest, passed as either a file or a json string""" return self._validate_manifest("output_manifest", source, **kwargs) @staticmethod def _get_cls(name, cls): - """ Getter that will return cls[name] if cls is a dict or cls otherwise - """ + """Getter that will return cls[name] if cls is a dict or cls otherwise""" return cls.get(name, None) if isinstance(cls, dict) else cls def validate(self, allow_missing=False, allow_extra=False, cls=None, **kwargs): - """ Validate strands from sources provided as keyword arguments + """Validate strands from sources provided as keyword arguments Usage: ``` @@ -361,13 +345,11 @@ def validate(self, allow_missing=False, allow_extra=False, cls=None, **kwargs): return sources def validate_strand(self, name, source, **kwargs): - """ Validates a single strand by name - """ + """Validates a single strand by name""" return self.validate({name: source}, **kwargs)[name] def prepare(self, *args, cls=None, **kwargs): - """ Prepares instance for strand data using a class map - """ + """Prepares instance for strand data using a class map""" prepared = {} for arg in args: if arg not in ALL_STRANDS: diff --git a/twined/utils/encoders.py b/twined/utils/encoders.py index 0799f54..e9749dc 100644 --- a/twined/utils/encoders.py +++ b/twined/utils/encoders.py @@ -7,7 +7,7 @@ class TwinedEncoder(json.JSONEncoder): - """ An encoder which will cope with serialising numpy arrays, ndarrays and matrices to JSON (in list form) + """An encoder which will cope with serialising numpy arrays, ndarrays and matrices to JSON (in list form) This is designed to work "out of the box" to help people serialise the outputs from twined applications. It does not require installation of numpy - it'll work fine if numpy is not present, so can be used in a versatile diff --git a/twined/utils/load_json.py b/twined/utils/load_json.py index 49bed2f..1290c98 100644 --- a/twined/utils/load_json.py +++ b/twined/utils/load_json.py @@ -12,7 +12,7 @@ def load_json(source, *args, **kwargs): - """ Loads json, automatically detecting whether the input is a valid filename, a string containing json data, + """Loads json, automatically detecting whether the input is a valid filename, a string containing json data, or a python dict already (in which case the result is returned directly). That makes this function suitable for use in a pipeline where it's not clear whether data has been loaded yet, or diff --git a/twined/utils/strings.py b/twined/utils/strings.py index 99b7fd6..c134a0b 100644 --- a/twined/utils/strings.py +++ b/twined/utils/strings.py @@ -1,6 +1,5 @@ def trim_suffix(text, suffix): - """ Strip a suffix from text, if it appears (otherwise return text unchanged) - """ + """Strip a suffix from text, if it appears (otherwise return text unchanged)""" if not text.endswith(suffix): return text return text[: len(text) - len(suffix)]