diff --git a/katka/constants.py b/katka/constants.py index 7026560..d9d6dd1 100644 --- a/katka/constants.py +++ b/katka/constants.py @@ -11,6 +11,11 @@ (PIPELINE_STATUS_SUCCESS, PIPELINE_STATUS_SUCCESS), ) +PIPELINE_FINAL_STATUSES = ( + PIPELINE_STATUS_SUCCESS, + PIPELINE_STATUS_FAILED, +) + STEP_STATUS_NOT_STARTED = 'not started' STEP_STATUS_IN_PROGRESS = 'in progress' STEP_STATUS_WAITING = 'waiting' # A step cannot proceed until some action is completed @@ -28,11 +33,18 @@ ) STEP_FINAL_STATUSES = (STEP_STATUS_SKIPPED, STEP_STATUS_FAILED, STEP_STATUS_SUCCESS) +STEP_EXECUTED_STATUSES = (STEP_STATUS_FAILED, STEP_STATUS_SUCCESS) -RELEASE_STATUS_OPEN = 'open' -RELEASE_STATUS_CLOSED = 'closed' + +RELEASE_STATUS_IN_PROGRESS = 'in progress' +RELEASE_STATUS_FAILED = 'failed' +RELEASE_STATUS_SUCCESS = 'success' RELEASE_STATUS_CHOICES = ( - (RELEASE_STATUS_OPEN, RELEASE_STATUS_OPEN), - (RELEASE_STATUS_CLOSED, RELEASE_STATUS_CLOSED), + (RELEASE_STATUS_IN_PROGRESS, RELEASE_STATUS_IN_PROGRESS), + (RELEASE_STATUS_FAILED, RELEASE_STATUS_FAILED), + (RELEASE_STATUS_SUCCESS, RELEASE_STATUS_SUCCESS), ) + +TAG_PRODUCTION_CHANGE_STARTED = "production_change_start" +TAG_PRODUCTION_CHANGE_ENDED = "production_change_end" diff --git a/katka/migrations/0022_change_release_status_fields.py b/katka/migrations/0022_change_release_status_fields.py new file mode 100644 index 0000000..a102c19 --- /dev/null +++ b/katka/migrations/0022_change_release_status_fields.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2019-10-21 07:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('katka', '0021_added_step_tags'), + ] + + operations = [ + migrations.AlterField( + model_name='scmrelease', + name='status', + field=models.CharField(choices=[('in progress', 'in progress'), ('failed', 'failed'), ('success', 'success')], default='in progress', max_length=30), + ), + ] diff --git a/katka/models.py b/katka/models.py index 8520696..1409993 100644 --- a/katka/models.py +++ b/katka/models.py @@ -6,7 +6,7 @@ from encrypted_model_fields.fields import EncryptedCharField from katka.auditedmodel import AuditedModel from katka.constants import ( - PIPELINE_STATUS_CHOICES, PIPELINE_STATUS_INITIALIZING, RELEASE_STATUS_CHOICES, RELEASE_STATUS_OPEN, + PIPELINE_STATUS_CHOICES, PIPELINE_STATUS_INITIALIZING, RELEASE_STATUS_CHOICES, RELEASE_STATUS_IN_PROGRESS, STEP_STATUS_CHOICES, STEP_STATUS_NOT_STARTED, ) from katka.fields import KatkaSlugField @@ -149,7 +149,7 @@ class Meta: public_identifier = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=255) - status = models.CharField(max_length=30, choices=RELEASE_STATUS_CHOICES, default=RELEASE_STATUS_OPEN) + status = models.CharField(max_length=30, choices=RELEASE_STATUS_CHOICES, default=RELEASE_STATUS_IN_PROGRESS) released = models.DateTimeField(null=True) from_hash = models.CharField(max_length=64) to_hash = models.CharField(max_length=64) diff --git a/katka/releases.py b/katka/releases.py new file mode 100644 index 0000000..459fcf4 --- /dev/null +++ b/katka/releases.py @@ -0,0 +1,95 @@ +import logging +from dataclasses import dataclass + +from katka import constants +from katka.fields import username_on_model +from katka.models import SCMPipelineRun, SCMRelease, SCMStepRun + +log = logging.getLogger('katka') + + +@dataclass +class StepsPreConditions: + start_tag_executed: bool + end_tag_executed: bool + success_status_between_start_end: list + + @property + def all_status_between_start_and_finish_are_success(self): + return all(self.success_status_between_start_end) + + @property + def start_and_finish_tags_were_executed(self): + return self.start_tag_executed and self.end_tag_executed + + +def create_release_if_necessary(pipeline): + release = _get_current_release(pipeline) + with username_on_model(SCMRelease, pipeline.modified_username): + if release is None: + release = SCMRelease.objects.create() + + release.scm_pipeline_runs.add(pipeline) + release.save() + + +def close_release_if_pipeline_finished(pipeline: SCMPipelineRun): + if pipeline.status not in constants.PIPELINE_FINAL_STATUSES: + log.debug(f'Pipeline {pipeline.public_identifier} not finished, doing nothing') + return None + + pre_conditions = _gather_steps_pre_conditions(pipeline) + if not pre_conditions.start_and_finish_tags_were_executed: + log.debug(f'Pipeline {pipeline.public_identifier} missing start and/or finish tags') + return None + + release = _get_current_release(pipeline) + if release and pre_conditions.all_status_between_start_and_finish_are_success: + release.status = constants.RELEASE_STATUS_SUCCESS + elif release: + log.debug(f'Pipeline {pipeline.public_identifier} contains step(s) with failed status') + release.status = constants.RELEASE_STATUS_FAILED + else: + log.warning(f'No open release for pipeline {pipeline.public_identifier}') + + with username_on_model(SCMRelease, pipeline.modified_username): + if release: + release.save() + + return release.status if release else None + + +def _gather_steps_pre_conditions(pipeline): + steps = SCMStepRun.objects.filter(scm_pipeline_run=pipeline).order_by("sequence_id") + is_start_tag_executed = False + is_end_tag_executed = False + success_status_between_start_end = [] + for step in steps: + if step.status not in constants.STEP_EXECUTED_STATUSES: + continue + if constants.TAG_PRODUCTION_CHANGE_STARTED in step.tags.split(" "): + is_start_tag_executed = True + + if is_start_tag_executed: + success_status_between_start_end.append(step.status == constants.STEP_STATUS_SUCCESS) + + if constants.TAG_PRODUCTION_CHANGE_ENDED in step.tags.split(" "): + is_end_tag_executed = True + break + + return StepsPreConditions(is_start_tag_executed, is_end_tag_executed, success_status_between_start_end) + + +def _get_current_release(pipeline): + releases = SCMRelease.objects.filter( + status=constants.RELEASE_STATUS_IN_PROGRESS, scm_pipeline_runs__application=pipeline.application + ) + release = None + if len(releases) == 0: + log.error(f'No open releases found for application {pipeline.application.pk}') + elif len(releases) > 1: + log.error(f'Multiple open releases found for application {pipeline.application.pk}, picking newest') + release = releases.order_by('-created').first() + else: + release = releases[0] + return release diff --git a/katka/signals.py b/katka/signals.py index 36efed3..21af6af 100644 --- a/katka/signals.py +++ b/katka/signals.py @@ -4,9 +4,10 @@ from django.db.models.signals import post_save from django.dispatch import receiver -from .constants import PIPELINE_STATUS_INITIALIZING, RELEASE_STATUS_OPEN, STEP_FINAL_STATUSES -from .fields import username_on_model -from .models import SCMPipelineRun, SCMRelease, SCMStepRun +from katka.constants import PIPELINE_STATUS_INITIALIZING, STEP_FINAL_STATUSES +from katka.fields import username_on_model +from katka.models import SCMPipelineRun, SCMStepRun +from katka.releases import close_release_if_pipeline_finished, create_release_if_necessary log = logging.getLogger('katka') @@ -35,6 +36,8 @@ def send_pipeline_change_notification(sender, **kwargs): pipeline = kwargs['instance'] if kwargs['created'] is True: create_release_if_necessary(pipeline) + else: + close_release_if_pipeline_finished(pipeline) if pipeline.status == PIPELINE_STATUS_INITIALIZING and kwargs['created'] is False: # Do not send notifications when the pipeline is initializing. While initializing, steps are created and @@ -48,20 +51,3 @@ def send_pipeline_change_notification(sender, **kwargs): session.post( settings.PIPELINE_CHANGE_NOTIFICATION_URL, json={'public_identifier': str(pipeline.public_identifier)} ) - - -def create_release_if_necessary(pipeline): - releases = SCMRelease.objects.filter( - status=RELEASE_STATUS_OPEN, scm_pipeline_runs__application=pipeline.application - ) - with username_on_model(SCMRelease, pipeline.modified_username): - if len(releases) == 0: - release = SCMRelease.objects.create() - elif len(releases) > 1: - log.error(f'Multiple open releases found for application {pipeline.application.pk}, picking newest') - release = releases.order_by('-created').first() - else: - release = releases[0] - - release.scm_pipeline_runs.add(pipeline) - release.save() diff --git a/tests/integration/conftest.py b/tests/conftest.py similarity index 74% rename from tests/integration/conftest.py rename to tests/conftest.py index 9122ac7..86c8c1a 100644 --- a/tests/integration/conftest.py +++ b/tests/conftest.py @@ -447,3 +447,109 @@ def not_my_metadata(not_my_application): meta.save() return meta + + +@pytest.fixture +def step_data(scm_pipeline_run): + + steps = [ + {"name": "step0", "stage": "prepare", "seq": "1.1-1", "status": "success", "tags": ""}, + {"name": "step1", "stage": "prepare", "seq": "1.2-1", "status": "success", "tags": ""}, + {"name": "step2", "stage": "deploy", "seq": "2.1-1", "status": "success", "tags": ""}, + {"name": "step3", "stage": "deploy", "seq": "2.2-1", "status": "success", "tags": ""}, + {"name": "step4", "stage": "deploy", "seq": "2.3-1", "status": "success", "tags": ""}, + {"name": "step5", "stage": "deploy", "seq": "2.4-1", "status": "success", "tags": ""}, + {"name": "step6", "stage": "deploy", "seq": "2.5-1", "status": "success", "tags": ""}, + ] + return steps + + +def _create_steps_from_dict(scm_pipeline_run, step_data): + saved_steps = [] + for step in step_data: + scm_step_run = models.SCMStepRun(slug=step["name"], name=step["name"], stage=step["stage"], + scm_pipeline_run=scm_pipeline_run, sequence_id=step["seq"], + status=step["status"], tags=step["tags"]) + + with username_on_model(models.SCMStepRun, 'initial'): + scm_step_run.save() + saved_steps.append(scm_step_run) + + return saved_steps + + +@pytest.fixture +def scm_step_run_success_list(scm_pipeline_run, step_data): + return _create_steps_from_dict(scm_pipeline_run, step_data) + + +@pytest.fixture +def scm_step_run_success_list_with_start_tag(scm_pipeline_run, step_data): + step_data[2]["tags"] = "production_change_start" + return _create_steps_from_dict(scm_pipeline_run, step_data) + + +@pytest.fixture +def scm_step_run_success_list_with_start_end_tags(scm_pipeline_run, step_data): + step_data[2]["tags"] = "production_change_start" + step_data[5]["tags"] = "production_change_end" + return _create_steps_from_dict(scm_pipeline_run, step_data) + + +@pytest.fixture +def scm_step_run_one_failed_step_list_with_start_end_tags(scm_pipeline_run, step_data): + step_data[2]["tags"] = "production_change_start" + step_data[4]["status"] = "failed" + step_data[5]["tags"] = "production_change_end" + return _create_steps_from_dict(scm_pipeline_run, step_data) + + +@pytest.fixture +def scm_step_run_one_failed_step_before_start_tag(scm_pipeline_run, step_data): + step_data[1]["status"] = "failed" + step_data[2]["status"] = "skipped" + step_data[3]["status"] = "skipped" + step_data[4]["status"] = "skipped" + step_data[6]["status"] = "skipped" + step_data[2]["tags"] = "production_change_start" + step_data[5]["tags"] = "production_change_end" + return _create_steps_from_dict(scm_pipeline_run, step_data) + + +@pytest.fixture +def scm_step_run_one_failed_step_after_end_tag(scm_pipeline_run, step_data): + step_data[2]["tags"] = "production_change_start" + step_data[5]["tags"] = "production_change_end" + step_data[6]["status"] = "failed" + return _create_steps_from_dict(scm_pipeline_run, step_data) + + +@pytest.fixture +def scm_pipeline_run_with_no_open_release(another_project, another_scm_repository, + scm_step_run_success_list_with_start_end_tags): + application = models.Application(project=another_project, + scm_repository=another_scm_repository, + name='Application 2', slug='APP2') + with username_on_model(models.Application, 'initial'): + application.save() + + pipeline_yaml = '''stages: + - release + + do-release: + stage: release + ''' + scm_pipeline_run = models.SCMPipelineRun(application=application, + pipeline_yaml=pipeline_yaml, + steps_total=5, + status="success", + commit_hash='4015B57A143AEC5156FD1444A017A32137A3FD0F') + with username_on_model(models.SCMPipelineRun, 'initial'): + scm_pipeline_run.save() + + with username_on_model(models.SCMStepRun, 'initial'): + for step in scm_step_run_success_list_with_start_end_tags: + step.scm_pipeline_run = scm_pipeline_run + step.save() + + return scm_pipeline_run diff --git a/tests/integration/test_scmrelease_view.py b/tests/integration/test_scmrelease_view.py index 8a0765c..a265971 100644 --- a/tests/integration/test_scmrelease_view.py +++ b/tests/integration/test_scmrelease_view.py @@ -64,7 +64,7 @@ def test_list(self, client, logged_in_user, scm_pipeline_run, scm_release): assert parsed[0]['released'] is None assert parsed[0]['from_hash'] == '577fe3f6a091aa4bad996623b1548b87f4f9c1f8' assert parsed[0]['to_hash'] == 'a49954f060b1b7605e972c9448a74d4067547443' - assert parsed[0]['status'] == 'open' + assert parsed[0]['status'] == 'in progress' assert len(parsed[0]['scm_pipeline_runs']) == 1 assert UUID(parsed[0]['scm_pipeline_runs'][0]) == scm_pipeline_run.public_identifier UUID(parsed[0]['public_identifier']) # should not raise @@ -81,7 +81,7 @@ def test_filtered_list(self, client, logged_in_user, scm_pipeline_run, scm_relea assert parsed[0]['released'] is None assert parsed[0]['from_hash'] == '100763d7144e1f993289bd528dc698dd3906a807' assert parsed[0]['to_hash'] == '38d72050370e6e0b43df649c9630f7135ef6de0d' - assert parsed[0]['status'] == 'open' + assert parsed[0]['status'] == 'in progress' assert len(parsed[0]['scm_pipeline_runs']) == 1 assert UUID(parsed[0]['scm_pipeline_runs'][0]) == another_scm_pipeline_run.public_identifier @@ -107,7 +107,7 @@ def test_get(self, client, logged_in_user, scm_pipeline_run, scm_release): assert parsed['released'] is None assert parsed['from_hash'] == '577fe3f6a091aa4bad996623b1548b87f4f9c1f8' assert parsed['to_hash'] == 'a49954f060b1b7605e972c9448a74d4067547443' - assert parsed['status'] == 'open' + assert parsed['status'] == 'in progress' assert len(parsed['scm_pipeline_runs']) == 1 assert UUID(parsed['scm_pipeline_runs'][0]) == scm_pipeline_run.public_identifier diff --git a/tests/unit/test_close_release.py b/tests/unit/test_close_release.py new file mode 100644 index 0000000..efb3e26 --- /dev/null +++ b/tests/unit/test_close_release.py @@ -0,0 +1,88 @@ +import pytest +from katka import constants +from katka.models import SCMRelease +from katka.signals import close_release_if_pipeline_finished + + +@pytest.mark.django_db +class TestCloseRelease: + + @staticmethod + def _assert_release_has_status(status): + assert SCMRelease.objects.count() == 1 + release = SCMRelease.objects.first() + assert release.status == status + + def test_do_nothing_if_pipeline_run_status_not_finished(self, scm_pipeline_run): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_IN_PROGRESS + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status is None + + # nothing has changed + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + def test_all_steps_are_success_but_no_start_tag_found(self, scm_pipeline_run, scm_step_run_success_list): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_SUCCESS + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status is None + + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + def test_all_steps_are_success_but_no_end_tag_found(self, scm_pipeline_run, + scm_step_run_success_list_with_start_tag): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_SUCCESS + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status is None + + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + def test_all_steps_are_success_but_no_open_release_found(self, scm_pipeline_run_with_no_open_release): + + result_status = close_release_if_pipeline_finished(scm_pipeline_run_with_no_open_release) + assert result_status is None + + def test_all_steps_are_success_with_start_and_end_tag(self, scm_pipeline_run, + scm_step_run_success_list_with_start_end_tags): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_SUCCESS + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status == constants.PIPELINE_STATUS_SUCCESS + + self._assert_release_has_status(constants.RELEASE_STATUS_SUCCESS) + + def test_one_failed_step_between_start_end_tags(self, scm_pipeline_run, + scm_step_run_one_failed_step_list_with_start_end_tags): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_FAILED + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status == constants.RELEASE_STATUS_FAILED + + self._assert_release_has_status(constants.RELEASE_STATUS_FAILED) + + def test_one_failed_step_before_start_tag(self, scm_pipeline_run, + scm_step_run_one_failed_step_before_start_tag): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_FAILED + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status is None + + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + def test_one_failed_step_after_end_tag(self, scm_pipeline_run, + scm_step_run_one_failed_step_after_end_tag): + self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS) + + scm_pipeline_run.status = constants.PIPELINE_STATUS_FAILED + result_status = close_release_if_pipeline_finished(scm_pipeline_run) + assert result_status == constants.RELEASE_STATUS_SUCCESS + + self._assert_release_has_status(constants.RELEASE_STATUS_SUCCESS)