diff --git a/foodx_devops_tools/deploy_me/_deployment.py b/foodx_devops_tools/deploy_me/_deployment.py index bc0c853..ccacc8c 100644 --- a/foodx_devops_tools/deploy_me/_deployment.py +++ b/foodx_devops_tools/deploy_me/_deployment.py @@ -168,14 +168,15 @@ def _construct_override_parameters( async def _do_step_deployment( this_step: ApplicationDeploymentDefinition, deployment_data: FlattenedDeployment, - puff_parameter_data: PuffMapPaths, + puff_parameter_paths: PuffMapPaths, this_context: str, enable_validation: bool, ) -> None: step_context = f"{this_context}.{this_step.name}" log.debug( - f"deployment_data.context, {step_context}, {str(deployment_data.context)}" # noqa: E501 + f"deployment_data.context, " + f"{step_context}, {str(deployment_data.context)}" ) log.debug( f"deployment_data.data, {step_context}, {str(deployment_data.data)}" @@ -187,12 +188,6 @@ async def _do_step_deployment( deployment_data.context.client, this_step.resource_group, ) - template_files = deployment_data.construct_deployment_paths( - this_step.arm_file, - this_step.puff_file, - puff_parameter_data[this_step.name], - ) - log.debug(f"template files, {template_files}") await login_service_principal(deployment_data.data.azure_credentials) if enable_validation: @@ -207,6 +202,13 @@ async def _do_step_deployment( template_parameters = deployment_data.construct_template_parameters() log.debug(f"template parameters, {step_context}, {template_parameters}") + template_files = deployment_data.construct_deployment_paths( + this_step.arm_file, + this_step.puff_file, + puff_parameter_paths[this_step.name], + ) + log.debug(f"template files, {template_files}") + deployment_files = await prepare_deployment_files( template_files, template_parameters, diff --git a/foodx_devops_tools/pipeline_config/views.py b/foodx_devops_tools/pipeline_config/views.py index 0413ebc..239a139 100644 --- a/foodx_devops_tools/pipeline_config/views.py +++ b/foodx_devops_tools/pipeline_config/views.py @@ -18,12 +18,7 @@ from foodx_devops_tools.azure.cloud import AzureCredentials from foodx_devops_tools.patterns import SubscriptionData from foodx_devops_tools.utilities.jinja2 import TemplateParameters -from foodx_devops_tools.utilities.templates import ( - JINJA_FILE_PREFIX, - ArmTemplateParameters, - ArmTemplates, - TemplateFiles, -) +from foodx_devops_tools.utilities.templates import TemplateFiles, TemplatePaths from ..deployment import DeploymentTuple from ._exceptions import PipelineViewError @@ -419,6 +414,21 @@ def construct_deployment_name(self: W, step_name: str) -> str: return result[0:64] + @staticmethod + def __construct_working_directory( + parent_dir: pathlib.Path, working_name: str + ) -> pathlib.Path: + working_dir = parent_dir / "working" / working_name + + return working_dir + + @staticmethod + def __encode_working_name(parameters: TemplateParameters) -> str: + """Construct a working directory name from template parameters.""" + # for now, use a simple predictable, non-empty value + value = "w" + return value + def construct_deployment_paths( self: W, specified_arm_file: typing.Optional[pathlib.Path], @@ -432,6 +442,7 @@ def construct_deployment_paths( specified_arm_file: specified_puff_file: target_arm_parameter_path: + working_name: Returns: Tuple of necessary paths. @@ -439,6 +450,8 @@ def construct_deployment_paths( PipelineViewError: If any errors occur due to undefined deployment data. """ + template_parameters = self.construct_template_parameters() + working_name = self.__encode_working_name(template_parameters) application_name = self.context.application_name frame_folder = self.data.frame_folder if not frame_folder: @@ -476,28 +489,27 @@ def construct_deployment_paths( ) log.debug(f"{puff_prompt}, {source_puff_path}") - working_dir = source_puff_path.parent - log.debug(f"working directory, {working_dir}") + working_dir = self.__construct_working_directory( + frame_folder, working_name + ) # Assume arm template parameters file has been specified with any sub # directories in it's path in puff_map.yml, so only frame_folder is # used here. - parameters_path = frame_folder / target_arm_parameter_path + parameters_path = working_dir / target_arm_parameter_path log.debug(f"arm parameters path, {parameters_path}") + log.debug(f"working directory, {working_dir}") target_arm_template_path = self.__screen_jinja_template( - source_arm_template_path, working_dir, "arm" - ) - target_puff_path = self.__screen_jinja_template( - source_puff_path, working_dir, "puff" + source_arm_template_path, working_dir ) template_files = TemplateFiles( - arm_template=ArmTemplates( - source=source_arm_template_path, target=target_arm_template_path + arm_template=TemplatePaths( + source=source_arm_template_path, + target=target_arm_template_path, ), - arm_template_parameters=ArmTemplateParameters( - source_puff=source_puff_path, - templated_puff=target_puff_path, + arm_template_parameters=TemplatePaths( + source=source_puff_path, target=parameters_path, ), ) @@ -506,18 +518,10 @@ def construct_deployment_paths( @staticmethod def __screen_jinja_template( - source_path: pathlib.Path, working_dir: pathlib.Path, keyword: str + source_path: pathlib.Path, working_dir: pathlib.Path ) -> pathlib.Path: - if source_path.name.startswith(JINJA_FILE_PREFIX): - # A jinja template file needs to have a target in the working dir. - target_prompt = f"jinja output {keyword} target" - target_path = working_dir / "{0}".format( - source_path.name.replace(JINJA_FILE_PREFIX, "") - ) - else: - target_prompt = f"{keyword} target unchanged from {keyword} source" - target_path = source_path - log.debug(f"{target_prompt}, {target_path}") + # A jinja template file needs to have a target in the working dir. + target_path = working_dir / "{0}".format(source_path.name) return target_path diff --git a/foodx_devops_tools/utilities/_exceptions.py b/foodx_devops_tools/utilities/_exceptions.py index 411774e..52cb710 100644 --- a/foodx_devops_tools/utilities/_exceptions.py +++ b/foodx_devops_tools/utilities/_exceptions.py @@ -14,3 +14,7 @@ class AnsibleVaultError(Exception): class CommandError(Exception): """Problem completing an external command run.""" + + +class TemplateError(Exception): + """Problem processing puff, jinja templates.""" diff --git a/foodx_devops_tools/utilities/exceptions.py b/foodx_devops_tools/utilities/exceptions.py index b2a7ead..f91628b 100644 --- a/foodx_devops_tools/utilities/exceptions.py +++ b/foodx_devops_tools/utilities/exceptions.py @@ -7,4 +7,8 @@ """Export utility related exceptions.""" -from ._exceptions import CommandError # noqa: F401 +from ._exceptions import ( # noqa: F401 + AnsibleVaultError, + CommandError, + TemplateError, +) diff --git a/foodx_devops_tools/utilities/templates.py b/foodx_devops_tools/utilities/templates.py index 390de5b..708cfe4 100644 --- a/foodx_devops_tools/utilities/templates.py +++ b/foodx_devops_tools/utilities/templates.py @@ -20,80 +20,26 @@ TemplateParameters, ) +from ._exceptions import TemplateError + log = logging.getLogger(__name__) JINJA_FILE_PREFIX = "jinja2." -class ArmTemplates(pydantic.BaseModel): +class TemplatePaths(pydantic.BaseModel): """Collection of file paths for ARM templates.""" source: pathlib.Path target: pathlib.Path - @pydantic.root_validator() - def check_jinja2_templating( - cls: pydantic.BaseModel, candidate: dict - ) -> dict: - """Check source target patterns when jinja2 templating is not needed.""" - source = candidate.get("source") - target = candidate.get("target") - if ( - source - and (not source.name.startswith(JINJA_FILE_PREFIX)) - and (source != target) - ): - message = ( - "source and target for non-jinja files must be " - "identical, {0}, {1}".format( - source, - target, - ) - ) - log.error(message) - raise ValueError(message) - - return candidate - - -class ArmTemplateParameters(pydantic.BaseModel): - """Collection of file paths for puff file and ARM template parameters.""" - - source_puff: pathlib.Path - templated_puff: pathlib.Path - target: pathlib.Path - - @pydantic.root_validator() - def check_jinja2_templating( - cls: pydantic.BaseModel, candidate: dict - ) -> dict: - """Check source target patterns when jinja2 templating is not needed.""" - source = candidate.get("source_puff") - target = candidate.get("templated_puff") - if ( - source - and (not source.name.startswith(JINJA_FILE_PREFIX)) - and (source != target) - ): - message = ( - "source and target for non-jinja files must be " - "identical, {0}, {1}".format( - source, - target, - ) - ) - log.error(message) - raise ValueError(message) - - return candidate - @dataclasses.dataclass class TemplateFiles: """Collection of file paths for template processing.""" - arm_template: ArmTemplates - arm_template_parameters: ArmTemplateParameters + arm_template: TemplatePaths + arm_template_parameters: TemplatePaths @dataclasses.dataclass @@ -126,9 +72,9 @@ def json_inlining(content: str) -> str: async def _apply_template( template_environment: FrameTemplates, source_file: pathlib.Path, - target_directory: pathlib.Path, + target_file: pathlib.Path, parameters: TemplateParameters, -) -> pathlib.Path: +) -> None: """ Apply frame-specific template and parameters ready for deployment. @@ -142,9 +88,6 @@ async def _apply_template( Returns: Target file path of the fulfilled template. """ - target_name = source_file.name.replace(JINJA_FILE_PREFIX, "") - target_file = target_directory / target_name - log.debug( "Applying jinja2 templating, {0} (source), " "{1} (destination)".format(source_file, target_file) @@ -153,77 +96,146 @@ async def _apply_template( source_file.name, target_file, parameters ) - return target_file + +def _verify_puff_target(file_path: pathlib.Path) -> None: + if not file_path.is_file(): + message = ( + f"Expected puff generated ARM template parameter file is " + f"missing, {file_path}" + ) + log.error(message) + raise TemplateError(message) -async def _apply_jinja2_file( - template_environment: FrameTemplates, - source_file: pathlib.Path, - target_directory: pathlib.Path, - parameters: TemplateParameters, -) -> pathlib.Path: - """Apply Jinja2 to an ARM template related file.""" - if source_file.name.startswith(JINJA_FILE_PREFIX): - templated_file = await _apply_template( - template_environment, source_file, target_directory, parameters +async def _prepare_working_directory(working_dir: pathlib.Path) -> None: + """ + Ensure that the working directory exists. + + Args: + working_dir: Expected path of working directory. + + Raises: + TemplateError: If the working directory path exists, but is not + actually a directory. + """ + if not working_dir.exists(): + working_dir.mkdir(parents=True, exist_ok=True) + elif not working_dir.is_dir(): + raise TemplateError( + f"working directory name exists but is not a directory, " + f"{working_dir}" ) - else: - templated_file = source_file - return templated_file + +def _log_arm_template_paths(arm_template: TemplatePaths) -> None: + log.debug(f"source_arm_template_path, {arm_template.source}") + log.debug(f"arm_target_file, {arm_template.target}") + + log.info(f"applying jinja2 to ARM template file, {arm_template.source}") + + +def _construct_arm_template_parameter_paths( + arm_template_parameters: TemplatePaths, +) -> pathlib.Path: + source_puff_file_path = arm_template_parameters.source + + # puffd_parameters_target_file: the expected arm template parameter file + # generated by puff. + puffd_parameters_target_file = arm_template_parameters.target + log.debug( + f"puffd_parameters_target_file json, {puffd_parameters_target_file}" + ) + parameters_target_dir = puffd_parameters_target_file.parent + log.debug(f"parameters_target_dir, {parameters_target_dir}") + + # jinjad_parameters_target_file: the expected arm template parameter file + # generated by jinja2 template processing. + jinjad_parameters_target_file = ( + parameters_target_dir / f"jinjad.{puffd_parameters_target_file.name}" + ) + log.debug( + f"jinjad_parameters_target_file json, {jinjad_parameters_target_file}" + ) + + log.info( + f"applying jinja2 to ARM template parameter file," + f" {source_puff_file_path}" + ) + + return jinjad_parameters_target_file async def prepare_deployment_files( template_files: TemplateFiles, parameters: TemplateParameters, ) -> ArmTemplateDeploymentFiles: - """Prepare final ARM template and parameter files for deployment.""" - source_arm_template_path = template_files.arm_template.source - source_puff_file_path = template_files.arm_template_parameters.source_puff - template_environment = FrameTemplates( - # folders containing _source_ files - list({source_arm_template_path.parent, source_puff_file_path.parent}) + """ + Prepare final ARM template and parameter files for deployment. + + Args: + template_files: Paths to source files for processing. + parameters: Parameters to be applied to templates. + + Returns: + Paths to ARM template and ARM template parameter files. + Raises: + TemplateError: If an error occurs during puff or template processing. + """ + arm_source = template_files.arm_template.source + arm_target = template_files.arm_template.target + _log_arm_template_paths(template_files.arm_template) + + # the puff YAML file. + parameters_source = template_files.arm_template_parameters.source + # the arm template parameters file generated from the puff run. + puffd_parameters_target = template_files.arm_template_parameters.target + # the arm template parameter file generated by jinja2 processing. + parameters_target = _construct_arm_template_parameter_paths( + template_files.arm_template_parameters ) - template_environment.environment.filters["json_inlining"] = json_inlining - arm_target_directory = template_files.arm_template.target.parent - log.debug(f"arm templating output target directory, {arm_target_directory}") - puff_target_directory = template_files.arm_template_parameters.target.parent - log.debug( - f"puff templating output target directory," f" {puff_target_directory}" + # folders containing _jinja template_ source files + template_paths = ( + [arm_source.parent, parameters_target.parent] + if arm_source.parent != parameters_target.parent + else [arm_source.parent] ) + log.debug(f"frame template paths, {template_paths}") + template_environment = FrameTemplates(template_paths) + template_environment.environment.filters["json_inlining"] = json_inlining - log.info(f"applying jinja2 to puff file, {source_puff_file_path}") - log.info( - f"applying jinja2 to ARM template file, {source_arm_template_path}" + await _prepare_working_directory(parameters_target.parent) + if parameters_target.parent != arm_target.parent: + # also prepare the distinct arm target directory + await _prepare_working_directory(arm_target.parent) + + # transform the puff file to arm template parameter json files. + await run_puff( + parameters_source, + False, + False, + disable_ascii_art=True, + output_dir=puffd_parameters_target.parent, ) - futures = await asyncio.gather( - _apply_jinja2_file( + _verify_puff_target(puffd_parameters_target) + + # now process jinja2 templates against JSON files. + await asyncio.gather( + _apply_template( template_environment, - source_puff_file_path, - puff_target_directory, + puffd_parameters_target, + parameters_target, parameters, ), - _apply_jinja2_file( + _apply_template( template_environment, - source_arm_template_path, - arm_target_directory, + arm_source, + arm_target, parameters, ), ) - templated_puff = futures[0] - templated_arm = futures[1] - # now transform the jinja2 processed puff file to arm template parameter - # json files. - await run_puff( - templated_puff, - False, - False, - disable_ascii_art=True, - output_dir=puff_target_directory, - ) result = ArmTemplateDeploymentFiles( - arm_template=templated_arm, - parameters=template_files.arm_template_parameters.target, + arm_template=arm_target, + parameters=parameters_target, ) return result diff --git a/tests/ci/unit_tests/deploy_me/deployment/test_deploy_application.py b/tests/ci/unit_tests/deploy_me/deployment/test_deploy_application.py index 2757f7b..8803b73 100644 --- a/tests/ci/unit_tests/deploy_me/deployment/test_deploy_application.py +++ b/tests/ci/unit_tests/deploy_me/deployment/test_deploy_application.py @@ -59,6 +59,10 @@ def prep_data(mock_async_method, mock_flattened_deployment): mock_async_method( "foodx_devops_tools.deploy_me._deployment.login_service_principal" ) + mock_async_method( + "foodx_devops_tools.utilities.templates._prepare_working_directory" + ) + mock_async_method("foodx_devops_tools.utilities.templates._apply_template") return mock_deploy, mock_puff, deployment_data, app_data @@ -112,7 +116,7 @@ async def check_auto_resource_group( class TestValidation(DeploymentChecks): @pytest.mark.asyncio async def test_static_resource_group( - self, default_override_parameters, prep_data + self, default_override_parameters, mock_verify_puff_target, prep_data ): enable_validation = True @@ -123,8 +127,8 @@ async def test_static_resource_group( expected_parameters = default_override_parameters(prep_data[2]) mock_deploy.assert_called_once_with( "c1-a1_group-123456", - pathlib.Path("some/path/a1.json"), - pathlib.Path("some/path/some/puff_map/path"), + pathlib.Path("some/path/working/w/a1.json"), + pathlib.Path("some/path/working/w/some/puff_map/jinjad.path"), "l1", "Incremental", AzureSubscriptionConfiguration(subscription_id="sys1_c1_r1a"), @@ -135,7 +139,7 @@ async def test_static_resource_group( @pytest.mark.asyncio async def test_auto_resource_group( - self, default_override_parameters, prep_data + self, default_override_parameters, mock_verify_puff_target, prep_data ): enable_validation = True @@ -146,8 +150,8 @@ async def test_auto_resource_group( expected_parameters = default_override_parameters(prep_data[2]) mock_deploy.assert_called_once_with( "c1-f1-a1-123456", - pathlib.Path("some/path/a1.json"), - pathlib.Path("some/path/some/puff_map/path"), + pathlib.Path("some/path/working/w/a1.json"), + pathlib.Path("some/path/working/w/some/puff_map/jinjad.path"), "l1", "Incremental", AzureSubscriptionConfiguration(subscription_id="sys1_c1_r1a"), @@ -160,7 +164,7 @@ async def test_auto_resource_group( class TestDeployment(DeploymentChecks): @pytest.mark.asyncio async def test_static_resource_group( - self, default_override_parameters, prep_data + self, default_override_parameters, mock_verify_puff_target, prep_data ): enable_validation = False @@ -171,8 +175,8 @@ async def test_static_resource_group( expected_parameters = default_override_parameters(prep_data[2]) mock_deploy.assert_called_once_with( "c1-a1_group", - pathlib.Path("some/path/a1.json"), - pathlib.Path("some/path/some/puff_map/path"), + pathlib.Path("some/path/working/w/a1.json"), + pathlib.Path("some/path/working/w/some/puff_map/jinjad.path"), "l1", "Incremental", AzureSubscriptionConfiguration(subscription_id="sys1_c1_r1a"), @@ -183,7 +187,7 @@ async def test_static_resource_group( @pytest.mark.asyncio async def test_auto_resource_group( - self, default_override_parameters, prep_data + self, default_override_parameters, mock_verify_puff_target, prep_data ): enable_validation = False @@ -194,8 +198,8 @@ async def test_auto_resource_group( expected_parameters = default_override_parameters(prep_data[2]) mock_deploy.assert_called_once_with( "c1-f1-a1", - pathlib.Path("some/path/a1.json"), - pathlib.Path("some/path/some/puff_map/path"), + pathlib.Path("some/path/working/w/a1.json"), + pathlib.Path("some/path/working/w/some/puff_map" "/jinjad.path"), "l1", "Incremental", AzureSubscriptionConfiguration(subscription_id="sys1_c1_r1a"), diff --git a/tests/ci/unit_tests/deploy_me/deployment/test_deploy_step.py b/tests/ci/unit_tests/deploy_me/deployment/test_deploy_step.py index 961423f..779555f 100644 --- a/tests/ci/unit_tests/deploy_me/deployment/test_deploy_step.py +++ b/tests/ci/unit_tests/deploy_me/deployment/test_deploy_step.py @@ -39,10 +39,12 @@ def mock_login(mock_async_method): @pytest.mark.asyncio async def test_clean( + mock_apply_template, mock_deploystep_context, mock_login, mock_rg_deploy, mock_run_puff, + mock_verify_puff_target, ): await _deploy_step(**mock_deploystep_context) @@ -52,10 +54,12 @@ async def test_clean( @pytest.mark.asyncio async def test_default_override_parameters( default_override_parameters, + mock_apply_template, mock_deploystep_context, mock_login, mock_rg_deploy, mock_run_puff, + mock_verify_puff_target, mocker, ): this_context = copy.deepcopy(mock_deploystep_context) @@ -82,10 +86,12 @@ async def test_default_override_parameters( @pytest.mark.asyncio async def test_secrets_enabled( default_override_parameters, + mock_apply_template, mock_deploystep_context, mock_login, mock_rg_deploy, mock_run_puff, + mock_verify_puff_target, mocker, ): this_context = copy.deepcopy(mock_deploystep_context) diff --git a/tests/ci/unit_tests/pipeline_config/test_checks.py b/tests/ci/unit_tests/pipeline_config/test_checks.py index 34847ff..571673e 100644 --- a/tests/ci/unit_tests/pipeline_config/test_checks.py +++ b/tests/ci/unit_tests/pipeline_config/test_checks.py @@ -21,7 +21,35 @@ log = logging.getLogger(__name__) -MOCK_SYSTEM_PATH = pathlib.Path("some/path") +@pytest.fixture() +def mock_file_exists(mock_async_method, mock_verify_puff_target): + def _apply(return_value=None, side_effect=None): + mock_async_method( + "foodx_devops_tools.pipeline_config._checks._file_exists", + return_value=return_value, + side_effect=side_effect, + ) + + return _apply + + +@pytest.fixture() +def path_check_mocks( + mock_apply_template, + mock_loads, + mock_results, + mock_run_puff_check, + mock_file_exists, +): + def _apply(return_value=None, side_effect=None): + mock_file_exists(return_value=return_value, side_effect=side_effect) + mock_loads(mock_results) + + mock_config = PipelineConfiguration.from_files(MOCK_PATHS, MOCK_SECRET) + + return mock_config + + return _apply class TestFileExists: @@ -48,28 +76,16 @@ async def test_false(self): class TestDoPathCheck: @pytest.mark.asyncio - async def test_clean( - self, mock_loads, mock_results, mock_async_method, mock_run_puff_check - ): - mock_async_method( - "foodx_devops_tools.pipeline_config._checks._file_exists", - return_value=True, - ) - mock_loads(mock_results) - mock_config = PipelineConfiguration.from_files(MOCK_PATHS, MOCK_SECRET) + async def test_clean(self, path_check_mocks): + mock_config = path_check_mocks(return_value=True) await do_path_check(mock_config) @pytest.mark.asyncio - async def test_missing_file( - self, mock_loads, mock_results, mock_async_method, mock_run_puff_check - ): - mock_async_method( - "foodx_devops_tools.pipeline_config._checks._file_exists", - side_effect=[True, False, True, True, True], + async def test_missing_file(self, path_check_mocks): + mock_config = path_check_mocks( + side_effect=[True, False, True, True, True] ) - mock_loads(mock_results) - mock_config = PipelineConfiguration.from_files(MOCK_PATHS, MOCK_SECRET) with pytest.raises( FileNotFoundError, match=r"files missing from deployment" @@ -77,15 +93,10 @@ async def test_missing_file( await do_path_check(mock_config) @pytest.mark.asyncio - async def test_multiple_missing_files( - self, mock_loads, mock_results, mock_async_method, mock_run_puff_check - ): - mock_async_method( - "foodx_devops_tools.pipeline_config._checks._file_exists", - side_effect=[False, False, False, False, False], + async def test_multiple_missing_files(self, path_check_mocks): + mock_config = path_check_mocks( + side_effect=[False, False, False, False, False] ) - mock_loads(mock_results) - mock_config = PipelineConfiguration.from_files(MOCK_PATHS, MOCK_SECRET) with pytest.raises( FileNotFoundError, match=r"files missing from deployment" @@ -95,22 +106,14 @@ async def test_multiple_missing_files( @pytest.mark.asyncio async def test_failed_check( self, - mock_loads, - mock_results, - mock_async_method, + path_check_mocks, mocker, - mock_run_puff_check, ): mocker.patch( "foodx_devops_tools.pipeline_config._checks._check_arm_files", side_effect=RuntimeError(), ) - mock_async_method( - "foodx_devops_tools.pipeline_config._checks._file_exists", - return_value=True, - ) - mock_loads(mock_results) - mock_config = PipelineConfiguration.from_files(MOCK_PATHS, MOCK_SECRET) + mock_config = path_check_mocks(return_value=True) with pytest.raises(RuntimeError): await do_path_check(mock_config) diff --git a/tests/ci/unit_tests/pipeline_config/views/test_deployment_paths.py b/tests/ci/unit_tests/pipeline_config/views/test_deployment_paths.py index ea087d7..cf498c4 100644 --- a/tests/ci/unit_tests/pipeline_config/views/test_deployment_paths.py +++ b/tests/ci/unit_tests/pipeline_config/views/test_deployment_paths.py @@ -11,11 +11,7 @@ import pytest from foodx_devops_tools.pipeline_config.views import FlattenedDeployment -from foodx_devops_tools.utilities.templates import ( - ArmTemplateParameters, - ArmTemplates, - TemplateFiles, -) +from foodx_devops_tools.utilities.templates import TemplateFiles, TemplatePaths @pytest.fixture() @@ -32,12 +28,6 @@ class TestConstructDeploymentPaths: MOCK_CONTEXT = "some_context" def _do_check(self, parameters: dict, under_test: FlattenedDeployment): - labels = [ - "source_arm_template", - "target_arm_template", - "puff", - "arm_parameters", - ] result = under_test.construct_deployment_paths(*parameters["input"]) assert result == parameters["expected"] @@ -45,23 +35,22 @@ def _do_check(self, parameters: dict, under_test: FlattenedDeployment): def test_none_arm_file_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file None, - # puff_file + # specified_puff_file None, - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some/arm_parameters.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/this_app.json"), - target=pathlib.Path("frame/folder/this_app.json"), + target=pathlib.Path("frame/folder/working/w/this_app.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/this_app.yml"), - templated_puff=pathlib.Path("frame/folder/this_app.yml"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/this_app.yml"), target=pathlib.Path( - "frame/folder/some/arm_parameters.json" + "frame/folder/working/w/some/arm_parameters.json" ), ), ), @@ -72,23 +61,23 @@ def test_none_arm_file_puff_file(self, mock_test_data): def test_arm_file_none_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file pathlib.Path("arm_file.json"), - # puff_file + # specified_puff_file None, - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.generated.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/arm_file.json"), - target=pathlib.Path("frame/folder/arm_file.json"), + target=pathlib.Path("frame/folder/working/w/arm_file.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/arm_file.yml"), - templated_puff=pathlib.Path("frame/folder/arm_file.yml"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/arm_file.yml"), target=pathlib.Path( - "frame/folder/some.generated.puff.file.json" + "frame/folder/working/w/some.generated.puff.file" + ".json" ), ), ), @@ -99,25 +88,22 @@ def test_arm_file_none_puff_file(self, mock_test_data): def test_pathed_arm_file_none_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file pathlib.Path("sub/arm_file.json"), - # puff_file + # specified_puff_file None, - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.generated.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/sub/arm_file.json"), - target=pathlib.Path("frame/folder/sub/arm_file.json"), + target=pathlib.Path("frame/folder/working/w/arm_file.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/sub/arm_file.yml"), - templated_puff=pathlib.Path( - "frame/folder/sub/arm_file.yml" - ), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/sub/arm_file.yml"), target=pathlib.Path( - "frame/folder/some.generated.puff.file.json" + "frame/folder/working/w/some.generated.puff.file.json" ), ), ), @@ -128,23 +114,22 @@ def test_pathed_arm_file_none_puff_file(self, mock_test_data): def test_pathed_arm_parameters_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file None, - # puff_file + # specified_puff_file None, - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("sub/some.generated.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/this_app.json"), - target=pathlib.Path("frame/folder/this_app.json"), + target=pathlib.Path("frame/folder/working/w/this_app.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/this_app.yml"), - templated_puff=pathlib.Path("frame/folder/this_app.yml"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/this_app.yml"), target=pathlib.Path( - "frame/folder/sub/some.generated.puff.file.json" + "frame/folder/working/w/sub/some.generated.puff.file.json" ), ), ), @@ -155,22 +140,23 @@ def test_pathed_arm_parameters_file(self, mock_test_data): def test_independent_arm_file_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file pathlib.Path("arm_file.json"), - # puff_file + # specified_puff_file pathlib.Path("puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/arm_file.json"), - target=pathlib.Path("frame/folder/arm_file.json"), + target=pathlib.Path("frame/folder/working/w/arm_file.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/puff_file.yml"), - templated_puff=pathlib.Path("frame/folder/puff_file.yml"), - target=pathlib.Path("frame/folder/some.puff.file.json"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/puff_file.yml"), + target=pathlib.Path( + "frame/folder/working/w/some.puff.file.json" + ), ), ), } @@ -180,24 +166,23 @@ def test_independent_arm_file_puff_file(self, mock_test_data): def test_pathed_independent_arm_file_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file pathlib.Path("arm_file.json"), - # puff_file + # specified_puff_file pathlib.Path("puff/puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/arm_file.json"), - target=pathlib.Path("frame/folder/arm_file.json"), + target=pathlib.Path("frame/folder/working/w/arm_file.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/puff/puff_file.yml"), - templated_puff=pathlib.Path( - "frame/folder/puff/puff_file.yml" + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/puff/puff_file.yml"), + target=pathlib.Path( + "frame/folder/working/w/some.puff.file.json" ), - target=pathlib.Path("frame/folder/some.puff.file.json"), ), ), } @@ -207,27 +192,26 @@ def test_pathed_independent_arm_file_puff_file(self, mock_test_data): def test_out_of_frame_nonjinja_arm_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file pathlib.Path("../common/files/arm_file.json"), - # puff_file + # specified_puff_file pathlib.Path("puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path( "frame/folder/../common/files/arm_file.json" ), + target=pathlib.Path("frame/folder/working/w/arm_file.json"), + ), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/puff_file.yml"), target=pathlib.Path( - "frame/folder/../common/files/arm_file.json" + "frame/folder/working/w/some.puff.file.json" ), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/puff_file.yml"), - templated_puff=pathlib.Path("frame/folder/puff_file.yml"), - target=pathlib.Path("frame/folder/some.puff.file.json"), - ), ), } @@ -236,24 +220,27 @@ def test_out_of_frame_nonjinja_arm_file(self, mock_test_data): def test_out_of_frame_jinja_arm_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file pathlib.Path("../common/files/jinja2.arm_file.json"), - # puff_file + # specified_puff_file pathlib.Path("puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path( "frame/folder/../common/files/jinja2.arm_file.json" ), - target=pathlib.Path("frame/folder/arm_file.json"), + target=pathlib.Path( + "frame/folder/working/w/jinja2.arm_file.json" + ), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("frame/folder/puff_file.yml"), - templated_puff=pathlib.Path("frame/folder/puff_file.yml"), - target=pathlib.Path("frame/folder/some.puff.file.json"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("frame/folder/puff_file.yml"), + target=pathlib.Path( + "frame/folder/working/w/some.puff.file.json" + ), ), ), } @@ -263,26 +250,25 @@ def test_out_of_frame_jinja_arm_file(self, mock_test_data): def test_out_of_frame_nonjinja_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file None, - # puff_file + # specified_puff_file pathlib.Path("../common/files/puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/this_app.json"), - target=pathlib.Path("frame/folder/this_app.json"), + target=pathlib.Path("frame/folder/working/w/this_app.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path( + arm_template_parameters=TemplatePaths( + source=pathlib.Path( "frame/folder/../common/files/puff_file.yml" ), - templated_puff=pathlib.Path( - "frame/folder/../common/files/puff_file.yml" + target=pathlib.Path( + "frame/folder/working/w/some.puff.file.json" ), - target=pathlib.Path("frame/folder/some.puff.file.json"), ), ), } @@ -292,26 +278,25 @@ def test_out_of_frame_nonjinja_puff_file(self, mock_test_data): def test_out_of_frame_jinja_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file None, - # puff_file + # specified_puff_file pathlib.Path("../common/files/jinja2.puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/this_app.json"), - target=pathlib.Path("frame/folder/this_app.json"), + target=pathlib.Path("frame/folder/working/w/this_app.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path( + arm_template_parameters=TemplatePaths( + source=pathlib.Path( "frame/folder/../common/files/jinja2.puff_file.yml" ), - templated_puff=pathlib.Path( - "frame/folder/../common/files/puff_file.yml" + target=pathlib.Path( + "frame/folder/working/w/some.puff.file.json" ), - target=pathlib.Path("frame/folder/some.puff.file.json"), ), ), } @@ -321,26 +306,25 @@ def test_out_of_frame_jinja_puff_file(self, mock_test_data): def test_pathed_jinja_puff_file(self, mock_test_data): parameters = { "input": [ - # arm_file + # specified_arm_file None, - # puff_file + # specified_puff_file pathlib.Path("sub/jinja2.puff_file.yml"), - # target_arm_parameters_file + # target_arm_parameter_path pathlib.Path("some.puff.file.json"), ], "expected": TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("frame/folder/this_app.json"), - target=pathlib.Path("frame/folder/this_app.json"), + target=pathlib.Path("frame/folder/working/w/this_app.json"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path( + arm_template_parameters=TemplatePaths( + source=pathlib.Path( "frame/folder/sub/jinja2.puff_file.yml" ), - templated_puff=pathlib.Path( - "frame/folder/sub/puff_file.yml" + target=pathlib.Path( + "frame/folder/working/w/some.puff.file.json" ), - target=pathlib.Path("frame/folder/some.puff.file.json"), ), ), } diff --git a/tests/ci/unit_tests/utilities/test_templates.py b/tests/ci/unit_tests/utilities/test_templates.py index 81b7dab..e700a15 100644 --- a/tests/ci/unit_tests/utilities/test_templates.py +++ b/tests/ci/unit_tests/utilities/test_templates.py @@ -10,15 +10,17 @@ import pytest from foodx_devops_tools.utilities.templates import ( - ArmTemplateParameters, - ArmTemplates, TemplateFiles, + TemplatePaths, + _apply_template, + _construct_arm_template_parameter_paths, + json_inlining, prepare_deployment_files, ) @pytest.fixture() -def mock_run(mock_async_method): +def mock_run(mock_async_method, mocker): mock_puff = mock_async_method( "foodx_devops_tools.utilities.templates.run_puff" ) @@ -26,10 +28,26 @@ def mock_run(mock_async_method): "foodx_devops_tools.utilities.templates.FrameTemplates" ".apply_template" ) + mocker.patch("pathlib.Path.is_file", return_value=True) return mock_puff, mock_template +class TestPrepareArmTemplateParameterPaths: + def test_clean(self): + mock_data = TemplatePaths( + source=pathlib.Path("source/puff/file.yml"), + target=pathlib.Path("target/json/file.json"), + ) + + result = _construct_arm_template_parameter_paths(mock_data) + + assert ( + result == mock_data.target.parent / f"jinjad." + f"{mock_data.target.name}" + ) + + class TestPrepareDeploymentFiles: MOCK_PARAMETERS = { "k1": "v1", @@ -38,15 +56,14 @@ class TestPrepareDeploymentFiles: @pytest.mark.asyncio async def test_clean(self, mock_run): - mock_path = pathlib.Path("some/path") + mock_path = pathlib.Path("some/file") mock_templates = TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=mock_path, target=mock_path, ), - arm_template_parameters=ArmTemplateParameters( - source_puff=mock_path, - templated_puff=mock_path, + arm_template_parameters=TemplatePaths( + source=mock_path, target=mock_path, ), ) @@ -57,7 +74,7 @@ async def test_clean(self, mock_run): ) assert result.arm_template == mock_path - assert result.parameters == mock_path + assert result.parameters == pathlib.Path("some/jinjad.file") mock_puff.assert_called_once_with( mock_path, False, @@ -67,16 +84,15 @@ async def test_clean(self, mock_run): ) @pytest.mark.asyncio - async def test_jinja2_puff(self, mock_run): + async def test_jinja2_puff(self, mock_run, mocker): mock_puff, mock_template = mock_run mock_templates = TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("some/source/file"), - target=pathlib.Path("some/source/file"), + target=pathlib.Path("some/target/file"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("some/jinja2.path.yml"), - templated_puff=pathlib.Path("some/target/path.yml"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("some/path.yml"), target=pathlib.Path("some/target/generated.json"), ), ) @@ -87,33 +103,42 @@ async def test_jinja2_puff(self, mock_run): ) assert result.arm_template == mock_templates.arm_template.target - assert ( - result.parameters == mock_templates.arm_template_parameters.target + assert result.parameters == pathlib.Path( + "some/target/jinjad.generated.json" ) mock_puff.assert_called_once_with( - mock_templates.arm_template_parameters.templated_puff, + mock_templates.arm_template_parameters.source, False, False, disable_ascii_art=True, output_dir=pathlib.Path("some/target"), ) - mock_template.assert_called_once_with( - mock_templates.arm_template_parameters.source_puff.name, - mock_templates.arm_template_parameters.templated_puff, - self.MOCK_PARAMETERS, + mock_template.assert_has_calls( + [ + mocker.call( + mock_templates.arm_template_parameters.target.name, + pathlib.Path("some/target/jinjad.generated.json"), + self.MOCK_PARAMETERS, + ), + mocker.call( + mock_templates.arm_template.source.name, + mock_templates.arm_template.target, + self.MOCK_PARAMETERS, + ), + ], + any_order=True, ) @pytest.mark.asyncio - async def test_jinja2_arm(self, mock_run): + async def test_jinja2_arm(self, mock_run, mocker): mock_puff, mock_template = mock_run mock_templates = TemplateFiles( - arm_template=ArmTemplates( + arm_template=TemplatePaths( source=pathlib.Path("some/jinja2.path"), target=pathlib.Path("some/target/path"), ), - arm_template_parameters=ArmTemplateParameters( - source_puff=pathlib.Path("some/path.yml"), - templated_puff=pathlib.Path("some/path.yml"), + arm_template_parameters=TemplatePaths( + source=pathlib.Path("some/path.yml"), target=pathlib.Path("some/target/generated.json"), ), ) @@ -124,18 +149,28 @@ async def test_jinja2_arm(self, mock_run): ) assert result.arm_template == mock_templates.arm_template.target - assert ( - result.parameters == mock_templates.arm_template_parameters.target + assert result.parameters == pathlib.Path( + "some/target/jinjad.generated.json" ) mock_puff.assert_called_once_with( - mock_templates.arm_template_parameters.templated_puff, + mock_templates.arm_template_parameters.source, False, False, disable_ascii_art=True, output_dir=pathlib.Path("some/target"), ) - mock_template.assert_called_once_with( - mock_templates.arm_template.source.name, - mock_templates.arm_template.target, - self.MOCK_PARAMETERS, + mock_template.assert_has_calls( + [ + mocker.call( + mock_templates.arm_template_parameters.target.name, + pathlib.Path("some/target/jinjad.generated.json"), + self.MOCK_PARAMETERS, + ), + mocker.call( + mock_templates.arm_template.source.name, + mock_templates.arm_template.target, + self.MOCK_PARAMETERS, + ), + ], + any_order=True, ) diff --git a/tests/conftest.py b/tests/conftest.py index ec21512..ba08309 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,11 +22,21 @@ from tests.ci.support.pipeline_config import MOCK_RESULTS, MOCK_TO +@pytest.fixture() +def mock_verify_puff_target(mocker): + mocker.patch("foodx_devops_tools.utilities.templates._verify_puff_target") + + @pytest.fixture() def mock_run_puff_check(mock_async_method): mock_async_method("foodx_devops_tools.utilities.templates.run_puff") +@pytest.fixture() +def mock_apply_template(mock_async_method): + mock_async_method("foodx_devops_tools.utilities.templates._apply_template") + + @pytest.fixture def apply_pipeline_config_test(mocker): def _apply(