diff --git a/foodx_devops_tools/pipeline_config/__init__.py b/foodx_devops_tools/pipeline_config/__init__.py index f886e80..237ae62 100644 --- a/foodx_devops_tools/pipeline_config/__init__.py +++ b/foodx_devops_tools/pipeline_config/__init__.py @@ -42,6 +42,10 @@ load_subscriptions, ) from .systems import SystemsDefinition, load_systems # noqa: F401 +from .template_context import ( # noqa: F401 + TemplateContext, + load_template_context, +) from .tenants import TenantsDefinition, load_tenants # noqa: F401 from .views import ( # noqa: F401 DeploymentContext, diff --git a/foodx_devops_tools/pipeline_config/_exceptions.py b/foodx_devops_tools/pipeline_config/_exceptions.py index a94969a..902180f 100644 --- a/foodx_devops_tools/pipeline_config/_exceptions.py +++ b/foodx_devops_tools/pipeline_config/_exceptions.py @@ -62,3 +62,7 @@ class PipelineViewError(Exception): class ClientsDefinitionError(Exception): """Problem loading client definitions.""" + + +class TemplateContextError(Exception): + """Problem loading template context variable definitions.""" diff --git a/foodx_devops_tools/pipeline_config/_loader.py b/foodx_devops_tools/pipeline_config/_loader.py index f2fa495..eb7174c 100644 --- a/foodx_devops_tools/pipeline_config/_loader.py +++ b/foodx_devops_tools/pipeline_config/_loader.py @@ -13,6 +13,15 @@ from ruamel.yaml import YAML # type: ignore +def load_yaml_data(file_path: pathlib.Path) -> dict: + """Acquire YAML data from a file.""" + with file_path.open(mode="r") as f: + loader = YAML(typ="safe") + yaml_data = loader.load(f) + + return yaml_data + + def load_configuration( file_path: pathlib.Path, data_model: typing.Type[pydantic.BaseModel], @@ -33,9 +42,7 @@ def load_configuration( Raises: error_type: If an error occurs loading the file. """ - with file_path.open(mode="r") as f: - loader = YAML(typ="safe") - yaml_data = loader.load(f) + yaml_data = load_yaml_data(file_path) try: result = data_model.parse_obj(yaml_data) diff --git a/foodx_devops_tools/pipeline_config/exceptions.py b/foodx_devops_tools/pipeline_config/exceptions.py index 5af739e..e5c1c13 100644 --- a/foodx_devops_tools/pipeline_config/exceptions.py +++ b/foodx_devops_tools/pipeline_config/exceptions.py @@ -19,5 +19,6 @@ ReleaseStatesDefinitionError, SubscriptionsDefinitionError, SystemsDefinitionError, + TemplateContextError, TenantsDefinitionError, ) diff --git a/foodx_devops_tools/pipeline_config/template_context.py b/foodx_devops_tools/pipeline_config/template_context.py new file mode 100644 index 0000000..1982b83 --- /dev/null +++ b/foodx_devops_tools/pipeline_config/template_context.py @@ -0,0 +1,102 @@ +# Copyright (c) 2021 Food-X Technologies +# +# This file is part of foodx_devops_tools. +# +# You should have received a copy of the MIT License along with +# foodx_devops_tools. If not, see . + +"""Template context variable deployment configuration I/O.""" + + +import logging +import pathlib +import typing + +import pydantic + +from ._exceptions import TemplateContextError +from ._loader import load_yaml_data + +log = logging.getLogger(__name__) + +ENTITY_NAME = "context" + +ValueType = typing.Dict[str, typing.Dict[str, typing.Any]] + + +class TemplateContext(pydantic.BaseModel): + """Define a collection of template context variables.""" + + context: ValueType + + +def _apply_existing_keys( + context_data: dict, yaml_data: dict, this_path: pathlib.Path +) -> None: + """Merge existing template context keys instead of over-write.""" + existing_keys = [ + x + for x in yaml_data[ENTITY_NAME].keys() + if x in context_data[ENTITY_NAME].keys() + ] + if existing_keys: + log.debug( + f"merging existing template context keys, {existing_keys}, " + f"{this_path}" + ) + for key in existing_keys: + context_data[ENTITY_NAME][key].update(yaml_data[ENTITY_NAME][key]) + else: + log.debug( + f"no existing template context keys to be merged, {this_path}" + ) + + +def _apply_nonexisting_keys( + context_data: dict, yaml_data: dict, this_path: pathlib.Path +) -> None: + """Add non-existing template context keys to the context.""" + nonexisting_keys = [ + x + for x in yaml_data[ENTITY_NAME].keys() + if x not in context_data[ENTITY_NAME].keys() + ] + if nonexisting_keys: + log.debug( + f"adding new template context keys, {nonexisting_keys}, " + f"{this_path}" + ) + for key in nonexisting_keys: + context_data[ENTITY_NAME][key] = yaml_data[ENTITY_NAME][key] + else: + log.debug(f"no new template context keys, {this_path}") + + +def load_template_context( + context_paths: typing.Set[pathlib.Path], +) -> TemplateContext: + """Load template context variables from the relevant directory.""" + try: + context_data: dict = { + ENTITY_NAME: dict(), + } + for this_path in context_paths: + if this_path.is_file(): + log.info("loading template context, {0}".format(this_path)) + yaml_data = load_yaml_data(this_path) + if ENTITY_NAME in yaml_data: + _apply_existing_keys(context_data, yaml_data, this_path) + _apply_nonexisting_keys(context_data, yaml_data, this_path) + else: + message = ( + f"template context object not present in " + f"file, {this_path}" + ) + log.error(message) + raise TemplateContextError(message) + + this_object = TemplateContext.parse_obj(context_data) + + return this_object + except (NotADirectoryError, FileNotFoundError) as e: + raise TemplateContextError(str(e)) from e diff --git a/tests/ci/unit_tests/pipeline_config/test_template_context.py b/tests/ci/unit_tests/pipeline_config/test_template_context.py new file mode 100644 index 0000000..5eb5062 --- /dev/null +++ b/tests/ci/unit_tests/pipeline_config/test_template_context.py @@ -0,0 +1,109 @@ +# Copyright (c) 2021 Food-X Technologies +# +# This file is part of foodx_devops_tools. +# +# You should have received a copy of the MIT License along with +# foodx_devops_tools. If not, see . + +import contextlib +import pathlib +import tempfile +import typing + +from foodx_devops_tools.pipeline_config import load_template_context + + +@contextlib.contextmanager +def context_files( + content: typing.Dict[str, typing.Dict[str, str]] +) -> typing.Generator[typing.List[pathlib.Path], None, None]: + dir_paths = set() + file_paths = set() + with tempfile.TemporaryDirectory() as base_path: + for this_dir, file_data in content.items(): + dir_path = pathlib.Path(base_path) / this_dir + dir_path.mkdir(parents=True) + dir_paths.add(dir_path) + for file_name, file_content in file_data.items(): + this_file = dir_path / file_name + with this_file.open("w") as f: + f.write(file_content) + + file_paths.add(this_file) + + yield dir_paths, file_paths + + +def test_load_files(): + file_text = { + "a": { + "f1": """--- +context: + s1: + s1k1: s1k1v + s1k2: s1k2v + s2: + s2k1: s2k1v +""", + "f2": """--- +context: + s3: + s3k1: s3k1v +""", + }, + } + with context_files(file_text) as (dir_paths, file_paths): + result = load_template_context(file_paths) + + assert len(result.context) == 3 + assert result.context == { + "s1": { + "s1k1": "s1k1v", + "s1k2": "s1k2v", + }, + "s2": {"s2k1": "s2k1v"}, + "s3": {"s3k1": "s3k1v"}, + } + + +def test_load_dirs(): + file_text = { + "a": { + "f1": """--- +context: + s1: + s1k1: s1k1v + s1k2: s1k2v + s2: + s2k1: s2k1v +""", + "f2": """--- +context: + s3: + s3k1: s3k1v +""", + }, + "b": { + "f1": """--- +context: + s1: + s1k3: s1k3v + s4: + s4k1: s4k1v +""", + }, + } + with context_files(file_text) as (dir_paths, file_paths): + result = load_template_context(file_paths) + + assert len(result.context) == 4 + assert result.context == { + "s1": { + "s1k1": "s1k1v", + "s1k2": "s1k2v", + "s1k3": "s1k3v", + }, + "s2": {"s2k1": "s2k1v"}, + "s3": {"s3k1": "s3k1v"}, + "s4": {"s4k1": "s4k1v"}, + }