implement template context loading (#100)

foodx_devops_tools/pipeline_config/__init__.py

@@ -42,6 +42,10 @@
     load_subscriptions,
 )
 from .systems import SystemsDefinition, load_systems  # noqa: F401
+from .template_context import (  # noqa: F401
+    TemplateContext,
+    load_template_context,
+)
 from .tenants import TenantsDefinition, load_tenants  # noqa: F401
 from .views import (  # noqa: F401
     DeploymentContext,

foodx_devops_tools/pipeline_config/_exceptions.py

@@ -62,3 +62,7 @@ class PipelineViewError(Exception):
 
 class ClientsDefinitionError(Exception):
     """Problem loading client definitions."""
+
+
+class TemplateContextError(Exception):
+    """Problem loading template context variable definitions."""

foodx_devops_tools/pipeline_config/_loader.py

@@ -13,6 +13,15 @@
 from ruamel.yaml import YAML  # type: ignore
 
 
+def load_yaml_data(file_path: pathlib.Path) -> dict:
+    """Acquire YAML data from a file."""
+    with file_path.open(mode="r") as f:
+        loader = YAML(typ="safe")
+        yaml_data = loader.load(f)
+
+    return yaml_data
+
+
 def load_configuration(
     file_path: pathlib.Path,
     data_model: typing.Type[pydantic.BaseModel],
@@ -33,9 +42,7 @@ def load_configuration(
     Raises:
         error_type: If an error occurs loading the file.
     """
-    with file_path.open(mode="r") as f:
-        loader = YAML(typ="safe")
-        yaml_data = loader.load(f)
+    yaml_data = load_yaml_data(file_path)
 
     try:
         result = data_model.parse_obj(yaml_data)

foodx_devops_tools/pipeline_config/exceptions.py

@@ -19,5 +19,6 @@
     ReleaseStatesDefinitionError,
     SubscriptionsDefinitionError,
     SystemsDefinitionError,
+    TemplateContextError,
     TenantsDefinitionError,
 )

foodx_devops_tools/pipeline_config/template_context.py

@@ -0,0 +1,102 @@
+#  Copyright (c) 2021 Food-X Technologies
+#
+#  This file is part of foodx_devops_tools.
+#
+#  You should have received a copy of the MIT License along with
+#  foodx_devops_tools. If not, see <https://opensource.org/licenses/MIT>.
+
+"""Template context variable deployment configuration I/O."""
+
+
+import logging
+import pathlib
+import typing
+
+import pydantic
+
+from ._exceptions import TemplateContextError
+from ._loader import load_yaml_data
+
+log = logging.getLogger(__name__)
+
+ENTITY_NAME = "context"
+
+ValueType = typing.Dict[str, typing.Dict[str, typing.Any]]
+
+
+class TemplateContext(pydantic.BaseModel):
+    """Define a collection of template context variables."""
+
+    context: ValueType
+
+
+def _apply_existing_keys(
+    context_data: dict, yaml_data: dict, this_path: pathlib.Path
+) -> None:
+    """Merge existing template context keys instead of over-write."""
+    existing_keys = [
+        x
+        for x in yaml_data[ENTITY_NAME].keys()
+        if x in context_data[ENTITY_NAME].keys()
+    ]
+    if existing_keys:
+        log.debug(
+            f"merging existing template context keys, {existing_keys}, "
+            f"{this_path}"
+        )
+        for key in existing_keys:
+            context_data[ENTITY_NAME][key].update(yaml_data[ENTITY_NAME][key])
+    else:
+        log.debug(
+            f"no existing template context keys to be merged, {this_path}"
+        )
+
+
+def _apply_nonexisting_keys(
+    context_data: dict, yaml_data: dict, this_path: pathlib.Path
+) -> None:
+    """Add non-existing template context keys to the context."""
+    nonexisting_keys = [
+        x
+        for x in yaml_data[ENTITY_NAME].keys()
+        if x not in context_data[ENTITY_NAME].keys()
+    ]
+    if nonexisting_keys:
+        log.debug(
+            f"adding new template context keys, {nonexisting_keys}, "
+            f"{this_path}"
+        )
+        for key in nonexisting_keys:
+            context_data[ENTITY_NAME][key] = yaml_data[ENTITY_NAME][key]
+    else:
+        log.debug(f"no new template context keys, {this_path}")
+
+
+def load_template_context(
+    context_paths: typing.Set[pathlib.Path],
+) -> TemplateContext:
+    """Load template context variables from the relevant directory."""
+    try:
+        context_data: dict = {
+            ENTITY_NAME: dict(),
+        }
+        for this_path in context_paths:
+            if this_path.is_file():
+                log.info("loading template context, {0}".format(this_path))
+                yaml_data = load_yaml_data(this_path)
+                if ENTITY_NAME in yaml_data:
+                    _apply_existing_keys(context_data, yaml_data, this_path)
+                    _apply_nonexisting_keys(context_data, yaml_data, this_path)
+                else:
+                    message = (
+                        f"template context object not present in "
+                        f"file, {this_path}"
+                    )
+                    log.error(message)
+                    raise TemplateContextError(message)
+
+        this_object = TemplateContext.parse_obj(context_data)
+
+        return this_object
+    except (NotADirectoryError, FileNotFoundError) as e:
+        raise TemplateContextError(str(e)) from e

tests/ci/unit_tests/pipeline_config/test_template_context.py

@@ -0,0 +1,109 @@
+#  Copyright (c) 2021 Food-X Technologies
+#
+#  This file is part of foodx_devops_tools.
+#
+#  You should have received a copy of the MIT License along with
+#  foodx_devops_tools. If not, see <https://opensource.org/licenses/MIT>.
+
+import contextlib
+import pathlib
+import tempfile
+import typing
+
+from foodx_devops_tools.pipeline_config import load_template_context
+
+
+@contextlib.contextmanager
+def context_files(
+    content: typing.Dict[str, typing.Dict[str, str]]
+) -> typing.Generator[typing.List[pathlib.Path], None, None]:
+    dir_paths = set()
+    file_paths = set()
+    with tempfile.TemporaryDirectory() as base_path:
+        for this_dir, file_data in content.items():
+            dir_path = pathlib.Path(base_path) / this_dir
+            dir_path.mkdir(parents=True)
+            dir_paths.add(dir_path)
+            for file_name, file_content in file_data.items():
+                this_file = dir_path / file_name
+                with this_file.open("w") as f:
+                    f.write(file_content)
+
+                file_paths.add(this_file)
+
+        yield dir_paths, file_paths
+
+
+def test_load_files():
+    file_text = {
+        "a": {
+            "f1": """---
+context:
+  s1:
+    s1k1: s1k1v
+    s1k2: s1k2v
+  s2:
+    s2k1: s2k1v
+""",
+            "f2": """---
+context:
+  s3:
+    s3k1: s3k1v
+""",
+        },
+    }
+    with context_files(file_text) as (dir_paths, file_paths):
+        result = load_template_context(file_paths)
+
+    assert len(result.context) == 3
+    assert result.context == {
+        "s1": {
+            "s1k1": "s1k1v",
+            "s1k2": "s1k2v",
+        },
+        "s2": {"s2k1": "s2k1v"},
+        "s3": {"s3k1": "s3k1v"},
+    }
+
+
+def test_load_dirs():
+    file_text = {
+        "a": {
+            "f1": """---
+context:
+  s1:
+    s1k1: s1k1v
+    s1k2: s1k2v
+  s2:
+    s2k1: s2k1v
+""",
+            "f2": """---
+context:
+  s3:
+    s3k1: s3k1v
+""",
+        },
+        "b": {
+            "f1": """---
+context:
+  s1:
+    s1k3: s1k3v
+  s4:
+    s4k1: s4k1v
+""",
+        },
+    }
+    with context_files(file_text) as (dir_paths, file_paths):
+        result = load_template_context(file_paths)
+
+    assert len(result.context) == 4
+    assert result.context == {
+        "s1": {
+            "s1k1": "s1k1v",
+            "s1k2": "s1k2v",
+            "s1k3": "s1k3v",
+        },
+        "s2": {"s2k1": "s2k1v"},
+        "s3": {"s3k1": "s3k1v"},
+        "s4": {"s4k1": "s4k1v"},
+    }