From eb93e0d114070d00b39304f017a1124b98ec23d2 Mon Sep 17 00:00:00 2001 From: Russell Date: Tue, 30 Nov 2021 14:19:48 -0500 Subject: [PATCH] Feature/rcs/fix template context (#110) * improve config validation debug logging * fix template context file acquisition --- foodx_devops_tools/pipeline_config/_checks.py | 16 +++- foodx_devops_tools/pipeline_config/_paths.py | 79 +++++++++++++------ .../pipeline_config/template_context.py | 1 + .../unit_tests/pipeline_config/test_paths.py | 48 +++++++++++ 4 files changed, 119 insertions(+), 25 deletions(-) create mode 100644 tests/ci/unit_tests/pipeline_config/test_paths.py diff --git a/foodx_devops_tools/pipeline_config/_checks.py b/foodx_devops_tools/pipeline_config/_checks.py index 9fd044a..ec00270 100644 --- a/foodx_devops_tools/pipeline_config/_checks.py +++ b/foodx_devops_tools/pipeline_config/_checks.py @@ -127,11 +127,23 @@ async def _prepare_deployment_files( puff_paths[structure_name].file, puff_parameter_paths[puff_map_structure_name].file, ) - log.debug(f"template files, {template_files}") + log.debug( + f"template files for configuration validation," + f" {puff_map_structure_name}, {template_files}" + ) + + template_parameters = ( + this_iteration.construct_template_parameters() + ) + log.debug( + f"template parameters applied to configuration " + f"validation," + f" {puff_map_structure_name}, {template_parameters}" + ) deployment_files = await prepare_deployment_files( template_files, - this_iteration.construct_template_parameters(), + template_parameters, ) templated_arm_files.append(deployment_files.arm_template) diff --git a/foodx_devops_tools/pipeline_config/_paths.py b/foodx_devops_tools/pipeline_config/_paths.py index 07246b8..1220c12 100644 --- a/foodx_devops_tools/pipeline_config/_paths.py +++ b/foodx_devops_tools/pipeline_config/_paths.py @@ -87,23 +87,8 @@ def from_paths( client and system. """ this_object = cls() - client_files = list() - for x in client_config.iterdir(): - if ( - x.is_file() - and (x.name in PIPELINE_CONFIG_FILES) - and (x.stem not in cls.CONFIG_SUBDIRS) - ): - log.info("adding client configuration file, {0}".format(x)) - setattr(this_object, x.stem, x) - client_files.append(x) - - system_files = list() - for x in system_config.iterdir(): - if x.is_file() and (x.name in PIPELINE_CONFIG_FILES): - log.info("adding system configuration file, {0}".format(x)) - setattr(this_object, x.stem, x) - system_files.append(x) + client_files = this_object.__acquire_client_files(client_config) + system_files = this_object.__acquire_system_files(system_config) if len(client_files + system_files) > len(PIPELINE_CONFIG_FILES): # must be duplicate files between the directories @@ -113,23 +98,71 @@ def from_paths( "Duplicate files between " "directories, {0}, {1}".format(client_config, system_config) ) + + this_object.static_secrets = cls.__acquire_static_secrets(client_config) + this_object.context = cls.__acquire_template_context( + client_config, system_config + ) + + return this_object + + @staticmethod + def __acquire_static_secrets( + client_config: pathlib.Path, + ) -> typing.Set[pathlib.Path]: secrets_path = client_config / "static_secrets" - this_object.static_secrets = cls.__acquire_subdir_files( + result = PipelineConfigurationPaths.__acquire_subdir_files( secrets_path, "static secrets" ) + return result + + @staticmethod + def __acquire_template_context( + client_config: pathlib.Path, system_config: pathlib.Path + ) -> typing.Set[pathlib.Path]: # template context could be located in either client or system config. context_client_path = client_config / "context" - this_object.context = cls.__acquire_subdir_files( - context_client_path, "client template context" + client_context_files = ( + PipelineConfigurationPaths.__acquire_subdir_files( + context_client_path, "client template context" + ) ) context_system_path = system_config / "context" - this_object.context.union( - cls.__acquire_subdir_files( + system_context_files = ( + PipelineConfigurationPaths.__acquire_subdir_files( context_system_path, "system template context" ) ) + result = client_context_files.union(system_context_files) + return result - return this_object + def __acquire_client_files( + self: T, client_config: pathlib.Path + ) -> typing.List[pathlib.Path]: + client_files = list() + for x in client_config.iterdir(): + if ( + x.is_file() + and (x.name in PIPELINE_CONFIG_FILES) + and (x.stem not in self.CONFIG_SUBDIRS) + ): + log.info("adding client configuration file, {0}".format(x)) + setattr(self, x.stem, x) + client_files.append(x) + + return client_files + + def __acquire_system_files( + self: T, system_config: pathlib.Path + ) -> typing.List[pathlib.Path]: + system_files = list() + for x in system_config.iterdir(): + if x.is_file() and (x.name in PIPELINE_CONFIG_FILES): + log.info("adding system configuration file, {0}".format(x)) + setattr(self, x.stem, x) + system_files.append(x) + + return system_files @staticmethod def __acquire_subdir_files( diff --git a/foodx_devops_tools/pipeline_config/template_context.py b/foodx_devops_tools/pipeline_config/template_context.py index 92b7524..ad4a816 100644 --- a/foodx_devops_tools/pipeline_config/template_context.py +++ b/foodx_devops_tools/pipeline_config/template_context.py @@ -38,6 +38,7 @@ def load_template_context( context_data: dict = { ENTITY_NAME: dict(), } + log.info(f"template context paths, {context_paths}") for this_path in context_paths: if this_path.is_file(): log.info("loading template context, {0}".format(this_path)) diff --git a/tests/ci/unit_tests/pipeline_config/test_paths.py b/tests/ci/unit_tests/pipeline_config/test_paths.py new file mode 100644 index 0000000..07184a9 --- /dev/null +++ b/tests/ci/unit_tests/pipeline_config/test_paths.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021 Food-X Technologies +# +# This file is part of foodx_devops_tools. +# +# You should have received a copy of the MIT License along with +# foodx_devops_tools. If not, see . + +import pathlib + +from foodx_devops_tools.pipeline_config import PipelineConfigurationPaths + + +class TestPipelineConfigurationPaths: + def test_template_context(self, mocker): + mock_config = pathlib.Path("mock_config") + mock_system = pathlib.Path("mock_system") + mocker.patch( + "foodx_devops_tools.pipeline_config" + "._paths.PipelineConfigurationPaths" + "._PipelineConfigurationPaths__acquire_static_secrets" + ) + mocker.patch( + "foodx_devops_tools.pipeline_config" + "._paths.PipelineConfigurationPaths" + "._PipelineConfigurationPaths__acquire_client_files" + ) + mocker.patch( + "foodx_devops_tools.pipeline_config" + "._paths.PipelineConfigurationPaths" + "._PipelineConfigurationPaths__acquire_system_files" + ) + mock_paths = [ + {pathlib.Path("one")}, + {pathlib.Path("two")}, + ] + mocker.patch( + "foodx_devops_tools.pipeline_config" + "._paths.PipelineConfigurationPaths" + "._PipelineConfigurationPaths__acquire_subdir_files", + side_effect=mock_paths, + ) + + under_test = PipelineConfigurationPaths.from_paths( + mock_config, mock_system + ) + + expected_context = mock_paths[0].union(mock_paths[1]) + assert under_test.context == expected_context