From 1fe4b6412db4c7eed97d7861249e364851731307 Mon Sep 17 00:00:00 2001 From: Russell Date: Thu, 18 Nov 2021 23:23:48 -0500 Subject: [PATCH] Feature/rcs/branch targets (#106) * add optional branch field to deployment subscription pipelien config * implement gitref_patterns for subscription filtering --- foodx_devops_tools/deploy_me/_main.py | 1 + foodx_devops_tools/pipeline_config/_checks.py | 1 + .../pipeline_config/deployments.py | 1 + foodx_devops_tools/pipeline_config/views.py | 60 +++- tests/ci/support/pipeline_config.py | 1 + .../pipeline_config/test_deployments.py | 62 ++++ .../views/test_deployment_view.py | 310 ++++++++++++++++++ .../pipeline_config/views/test_views.py | 65 ---- tests/conftest.py | 1 + 9 files changed, 427 insertions(+), 75 deletions(-) create mode 100644 tests/ci/unit_tests/pipeline_config/views/test_deployment_view.py diff --git a/foodx_devops_tools/deploy_me/_main.py b/foodx_devops_tools/deploy_me/_main.py index 41912ba..e4113d2 100644 --- a/foodx_devops_tools/deploy_me/_main.py +++ b/foodx_devops_tools/deploy_me/_main.py @@ -253,6 +253,7 @@ def deploy_me( commit_sha = get_sha() base_context = DeploymentContext( commit_sha=commit_sha, + git_ref=git_ref, pipeline_id=pipeline_id, release_id=release_id, release_state=release_state.name, diff --git a/foodx_devops_tools/pipeline_config/_checks.py b/foodx_devops_tools/pipeline_config/_checks.py index e241e87..9fd044a 100644 --- a/foodx_devops_tools/pipeline_config/_checks.py +++ b/foodx_devops_tools/pipeline_config/_checks.py @@ -93,6 +93,7 @@ async def _prepare_deployment_files( for release_state in pipeline_configuration.release_states: base_context = DeploymentContext( commit_sha="abc123", + git_ref=None, pipeline_id="000", release_id="0.0.0+local", release_state=release_state, diff --git a/foodx_devops_tools/pipeline_config/deployments.py b/foodx_devops_tools/pipeline_config/deployments.py index aa93942..087959f 100644 --- a/foodx_devops_tools/pipeline_config/deployments.py +++ b/foodx_devops_tools/pipeline_config/deployments.py @@ -28,6 +28,7 @@ class DeploymentLocations(pydantic.BaseModel): class DeploymentSubscriptionReference(pydantic.BaseModel): """A subscription reference in a deployment definition.""" + gitref_patterns: typing.Optional[typing.List[str]] locations: typing.List[DeploymentLocations] root_fqdn: str diff --git a/foodx_devops_tools/pipeline_config/views.py b/foodx_devops_tools/pipeline_config/views.py index 6debf6d..f095028 100644 --- a/foodx_devops_tools/pipeline_config/views.py +++ b/foodx_devops_tools/pipeline_config/views.py @@ -46,6 +46,7 @@ class DeploymentContext: """Deployment context data expected to be applied to resource tags.""" commit_sha: str + git_ref: typing.Optional[str] pipeline_id: str release_id: str release_state: str @@ -62,12 +63,14 @@ class DeploymentContext: def __init__( self: Y, commit_sha: str, + git_ref: typing.Optional[str], pipeline_id: str, release_id: str, release_state: str, ) -> None: """Construct ``DeploymentContext`` object.""" self.commit_sha = commit_sha + self.git_ref = git_ref self.pipeline_id = pipeline_id self.release_id = release_id self.release_state = release_state @@ -647,20 +650,57 @@ def __init__( self._validate_deployment_tuple() + def __matched_subscription_patterns( + self: U, + subscription_name: str, + gitref_patterns: typing.Optional[typing.List[str]], + ) -> bool: + """ + Check that the subscription matches any specified patterns. + + If a git reference has not been specified then no conditioning is + applied and any specified subscriptions will be considered a "match". + """ + result = True + this_ref = self.release_view.deployment_context.git_ref + if this_ref and gitref_patterns: + not_match = [ + (re.match(x, this_ref) is None) for x in gitref_patterns + ] + if all(not_match): + result = False + + return result + @property def subscriptions(self: U) -> typing.List[SubscriptionView]: - """Provide the subscriptions in this deployment.""" + """ + Provide the subscriptions in this deployment. + + Qualifies subscriptions both by their presence in the deployment + definitions for the project, and the optional branch patterns + specified for the subscription. + + Returns: + List of subscription views in this deployment. + """ result: typing.List[SubscriptionView] = list() - if ( - str(self.deployment_tuple) - in self.release_view.configuration.deployments.deployment_tuples - ): - for ( - this_subscription - ) in self.release_view.configuration.deployments.deployment_tuples[ - str(self.deployment_tuple) + this_id = str(self.deployment_tuple) + deployment_ids = ( + self.release_view.configuration.deployments.deployment_tuples + ) + if this_id in deployment_ids: + for subscription_name in deployment_ids[ + this_id ].subscriptions.keys(): - result.append(SubscriptionView(self, this_subscription)) + this_subscription = deployment_ids[this_id].subscriptions[ + subscription_name + ] + gitref_patterns = this_subscription.gitref_patterns + if self.__matched_subscription_patterns( + subscription_name, gitref_patterns + ): + result.append(SubscriptionView(self, subscription_name)) return result diff --git a/tests/ci/support/pipeline_config.py b/tests/ci/support/pipeline_config.py index ce0162e..3d42572 100644 --- a/tests/ci/support/pipeline_config.py +++ b/tests/ci/support/pipeline_config.py @@ -22,6 +22,7 @@ MOCK_CONTEXT = DeploymentContext( commit_sha="abc123", + git_ref="refs/heads/some/branch", pipeline_id="12345", release_id="0.0.0-dev.3", release_state="r1", diff --git a/tests/ci/unit_tests/pipeline_config/test_deployments.py b/tests/ci/unit_tests/pipeline_config/test_deployments.py index 36088bf..a3276f5 100644 --- a/tests/ci/unit_tests/pipeline_config/test_deployments.py +++ b/tests/ci/unit_tests/pipeline_config/test_deployments.py @@ -209,3 +209,65 @@ def test_empty_list_raises(apply_deployments_test): match=r"Error validating deployments definition", ): apply_deployments_test(file_text) + + +def test_branch_option_absent(apply_deployments_test): + file_text = """ +--- +deployments: + deployment_tuples: + name: + subscriptions: + some-name: + locations: + - primary: ploc1 + secondary: sloc1 + - primary: ploc2 + root_fqdn: some.where + url_endpoints: ["a","b"] +""" + + result = apply_deployments_test(file_text) + + assert ( + result.deployments.deployment_tuples["name"] + .subscriptions["some-name"] + .gitref_patterns + is None + ) + + +def test_branch_option_present(apply_deployments_test): + file_text = """ +--- +deployments: + deployment_tuples: + name: + subscriptions: + some-name: + gitref_patterns: + - main + locations: + - primary: ploc1 + secondary: sloc1 + - primary: ploc2 + root_fqdn: some.where + url_endpoints: ["a","b"] +""" + + result = apply_deployments_test(file_text) + + assert ( + len( + result.deployments.deployment_tuples["name"] + .subscriptions["some-name"] + .gitref_patterns + ) + == 1 + ) + assert ( + result.deployments.deployment_tuples["name"] + .subscriptions["some-name"] + .gitref_patterns[0] + == "main" + ) diff --git a/tests/ci/unit_tests/pipeline_config/views/test_deployment_view.py b/tests/ci/unit_tests/pipeline_config/views/test_deployment_view.py new file mode 100644 index 0000000..25e50af --- /dev/null +++ b/tests/ci/unit_tests/pipeline_config/views/test_deployment_view.py @@ -0,0 +1,310 @@ +# Copyright (c) 2021 Food-X Technologies +# +# This file is part of foodx_devops_tools. +# +# You should have received a copy of the MIT License along with +# foodx_devops_tools. If not, see . + +import copy + +import pytest + +from foodx_devops_tools.pipeline_config.exceptions import PipelineViewError +from foodx_devops_tools.pipeline_config.views import ( + DeploymentContext, + DeploymentTuple, + DeploymentView, + ReleaseView, +) +from tests.ci.support.pipeline_config import MOCK_CONTEXT, MOCK_RESULTS + + +@pytest.fixture() +def mock_context() -> DeploymentContext: + this_context = copy.deepcopy(MOCK_CONTEXT) + this_context.release_state = "r2" + + return this_context + + +@pytest.fixture() +def this_deployment_view(mock_pipeline_config): + def _apply(mock_results: dict, this_context: DeploymentContext): + expected_state = DeploymentTuple( + client="c1", release_state="r1", system="sys1" + ) + release_view = ReleaseView( + mock_pipeline_config(mock_results), this_context + ) + under_test = DeploymentView(release_view, expected_state) + + return under_test + + return _apply + + +class TestDeploymentView: + def test_clean(self, mock_context, mock_pipeline_config): + expected_state = DeploymentTuple( + client="c1", release_state="r1", system="sys1" + ) + release_view = ReleaseView(mock_pipeline_config(), mock_context) + under_test = DeploymentView(release_view, expected_state) + + assert under_test.release_view == release_view + assert under_test.deployment_tuple == expected_state + + def test_bad_deployment_state_raises( + self, mock_context, mock_pipeline_config + ): + release_view = ReleaseView(mock_pipeline_config(), mock_context) + with pytest.raises(PipelineViewError, match=r"^Bad client"): + DeploymentView( + release_view, + DeploymentTuple( + client="bad_client", release_state="r2", system="sys2" + ), + ) + with pytest.raises(PipelineViewError, match=r"^Bad release state"): + DeploymentView( + release_view, + DeploymentTuple( + client="c1", release_state="bad_release", system="sys2" + ), + ) + with pytest.raises(PipelineViewError, match=r"^Bad system"): + DeploymentView( + release_view, + DeploymentTuple( + client="c1", release_state="r2", system="bad_system" + ), + ) + + def test_subscriptions_clean(self, mock_context, mock_pipeline_config): + expected_state = DeploymentTuple( + client="c1", release_state="r1", system="sys1" + ) + release_view = ReleaseView(mock_pipeline_config(), mock_context) + under_test = DeploymentView(release_view, expected_state) + + assert under_test.release_view == release_view + assert under_test.deployment_tuple == expected_state + + assert under_test.subscriptions + + def test_subscriptions_empty(self, mock_context, mock_pipeline_config): + this_context = copy.deepcopy(MOCK_CONTEXT) + this_context.release_state = "r2" + release_view = ReleaseView(mock_pipeline_config(), this_context) + under_test = DeploymentView( + release_view, + DeploymentTuple(client="c2", release_state="r2", system="sys2"), + ) + + assert not under_test.subscriptions + + +@pytest.fixture() +def deployment_view_config(): + mock_results = copy.deepcopy(MOCK_RESULTS) + mock_results["puff_map"] = { + "frames": { + "f1": { + "applications": { + "a1": { + "arm_parameters_files": { + "r1": { + "sys1_c1_r1a": { + "a1l1": "some/puff_map/path", + }, + "sys1_c1_r1b": { + "a1l1": "some/puff_map/path", + }, + "sys1_c1_r1c": { + "a1l1": "some/puff_map/path", + }, + }, + }, + }, + }, + }, + }, + } + mock_results["subscriptions"] = { + "sys1_c1_r1a": { + "ado_service_connection": "some-name", + "azure_id": "abc123", + "tenant": "t1", + }, + "sys1_c1_r1b": { + "ado_service_connection": "some-name", + "azure_id": "abc123", + "tenant": "t1", + }, + "sys1_c1_r1c": { + "ado_service_connection": "some-name", + "azure_id": "abc123", + "tenant": "t1", + }, + } + mock_results["service_principals"] = { + "sys1_c1_r1a": { + "id": "12345", + "secret": "verysecret", + "name": "sp_name", + }, + "sys1_c1_r1b": { + "id": "12345", + "secret": "verysecret", + "name": "sp_name", + }, + "sys1_c1_r1c": { + "id": "12345", + "secret": "verysecret", + "name": "sp_name", + }, + } + mock_results["static_secrets"] = { + "sys1_c1_r1a": {"k1": "k1v1"}, + "sys1_c1_r1b": {"k1": "k1v2"}, + "sys1_c1_r1c": {"k1": "k1v2"}, + } + + return mock_results + + +class TestSubscriptionConstraints: + def test_no_constraint( + self, mock_context, deployment_view_config, this_deployment_view + ): + deployment_view_config["deployments"] = { + "deployment_tuples": { + "sys1-c1-r1": { + "subscriptions": { + "sys1_c1_r1a": { + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "this.sub", + }, + "sys1_c1_r1b": { + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "other.sub", + }, + }, + }, + }, + "url_endpoints": ["a", "p"], + } + under_test = this_deployment_view(deployment_view_config, mock_context) + this_subscriptions = under_test.subscriptions + + assert len(under_test.subscriptions) == 2 + expected_names = {"sys1_c1_r1a", "sys1_c1_r1b"} + assert { + this_subscriptions[x].subscription_name + for x in range(len(this_subscriptions)) + } == expected_names + + def test_plain_text( + self, mock_context, deployment_view_config, this_deployment_view + ): + mock_context.git_ref = "b1" + deployment_view_config["deployments"] = { + "deployment_tuples": { + "sys1-c1-r1": { + "subscriptions": { + "sys1_c1_r1a": { + "gitref_patterns": ["b1"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + "sys1_c1_r1b": { + "gitref_patterns": ["b2"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + }, + }, + }, + "url_endpoints": ["a", "p"], + } + under_test = this_deployment_view(deployment_view_config, mock_context) + this_subscriptions = under_test.subscriptions + + assert len(under_test.subscriptions) == 1 + expected_names = {"sys1_c1_r1a"} + assert { + this_subscriptions[x].subscription_name + for x in range(len(this_subscriptions)) + } == expected_names + + def test_regex( + self, mock_context, deployment_view_config, this_deployment_view + ): + mock_context.git_ref = "b3/some/branch" + deployment_view_config["deployments"] = { + "deployment_tuples": { + "sys1-c1-r1": { + "subscriptions": { + "sys1_c1_r1a": { + "gitref_patterns": ["^b[2-4]/.*"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + "sys1_c1_r1b": { + "gitref_patterns": ["b2"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + }, + }, + }, + "url_endpoints": ["a", "p"], + } + under_test = this_deployment_view(deployment_view_config, mock_context) + this_subscriptions = under_test.subscriptions + + assert len(under_test.subscriptions) == 1 + expected_names = {"sys1_c1_r1a"} + assert { + this_subscriptions[x].subscription_name + for x in range(len(this_subscriptions)) + } == expected_names + + def test_multiple( + self, mock_context, deployment_view_config, this_deployment_view + ): + mock_context.git_ref = "c" + deployment_view_config["deployments"] = { + "deployment_tuples": { + "sys1-c1-r1": { + "subscriptions": { + "sys1_c1_r1a": { + "gitref_patterns": ["^b[2-4]/.*", "c"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + "sys1_c1_r1b": { + "gitref_patterns": ["c"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + "sys1_c1_r1c": { + # this subscription should be excluded + "gitref_patterns": ["d"], + "locations": [{"primary": "l1"}, {"primary": "l2"}], + "root_fqdn": "some.where", + }, + }, + }, + }, + "url_endpoints": ["a", "p"], + } + under_test = this_deployment_view(deployment_view_config, mock_context) + this_subscriptions = under_test.subscriptions + + assert len(under_test.subscriptions) == 2 + expected_names = {"sys1_c1_r1a", "sys1_c1_r1b"} + assert { + this_subscriptions[x].subscription_name + for x in range(len(this_subscriptions)) + } == expected_names diff --git a/tests/ci/unit_tests/pipeline_config/views/test_views.py b/tests/ci/unit_tests/pipeline_config/views/test_views.py index 8b2c211..6489909 100644 --- a/tests/ci/unit_tests/pipeline_config/views/test_views.py +++ b/tests/ci/unit_tests/pipeline_config/views/test_views.py @@ -186,71 +186,6 @@ def test_bad_subscription_raises(self, mock_pipeline_config): ) -class TestDeploymentView: - def test_clean(self, mock_pipeline_config): - expected_state = DeploymentTuple( - client="c1", release_state="r2", system="sys1" - ) - this_context = copy.deepcopy(MOCK_CONTEXT) - this_context.release_state = "r2" - release_view = ReleaseView(mock_pipeline_config(), this_context) - under_test = DeploymentView(release_view, expected_state) - - assert under_test.release_view == release_view - assert under_test.deployment_tuple == expected_state - - def test_bad_deployment_state_raises(self, mock_pipeline_config): - this_context = copy.deepcopy(MOCK_CONTEXT) - this_context.release_state = "r2" - release_view = ReleaseView(mock_pipeline_config(), this_context) - with pytest.raises(PipelineViewError, match=r"^Bad client"): - DeploymentView( - release_view, - DeploymentTuple( - client="bad_client", release_state="r2", system="sys2" - ), - ) - with pytest.raises(PipelineViewError, match=r"^Bad release state"): - DeploymentView( - release_view, - DeploymentTuple( - client="c1", release_state="bad_release", system="sys2" - ), - ) - with pytest.raises(PipelineViewError, match=r"^Bad system"): - DeploymentView( - release_view, - DeploymentTuple( - client="c1", release_state="r2", system="bad_system" - ), - ) - - def test_subscriptions_clean(self, mock_pipeline_config): - expected_state = DeploymentTuple( - client="c1", release_state="r1", system="sys1" - ) - this_context = copy.deepcopy(MOCK_CONTEXT) - this_context.release_state = "r2" - release_view = ReleaseView(mock_pipeline_config(), this_context) - under_test = DeploymentView(release_view, expected_state) - - assert under_test.release_view == release_view - assert under_test.deployment_tuple == expected_state - - assert under_test.subscriptions - - def test_subscriptions_empty(self, mock_pipeline_config): - this_context = copy.deepcopy(MOCK_CONTEXT) - this_context.release_state = "r2" - release_view = ReleaseView(mock_pipeline_config(), this_context) - under_test = DeploymentView( - release_view, - DeploymentTuple(client="c2", release_state="r2", system="sys2"), - ) - - assert not under_test.subscriptions - - class TestReleaseView: def test_clean(self, mock_pipeline_config): this_context = copy.deepcopy(MOCK_CONTEXT) diff --git a/tests/conftest.py b/tests/conftest.py index ba08309..f7a8ab0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,6 +65,7 @@ def _apply(mock_data=copy.deepcopy(MOCK_RESULTS)) -> PipelineConfiguration: def mock_flattened_deployment(mock_pipeline_config): base_context = DeploymentContext( commit_sha="abc123", + git_ref="refs/heads/this/branch", pipeline_id="123456", release_id="3.1.4+local", release_state="r1",