OPT: add close release if pipeline finished

katka/constants.py

@@ -11,6 +11,11 @@
     (PIPELINE_STATUS_SUCCESS, PIPELINE_STATUS_SUCCESS),
 )
 
+PIPELINE_FINAL_STATUSES = (
+    PIPELINE_STATUS_SUCCESS,
+    PIPELINE_STATUS_FAILED,
+)
+
 STEP_STATUS_NOT_STARTED = 'not started'
 STEP_STATUS_IN_PROGRESS = 'in progress'
 STEP_STATUS_WAITING = 'waiting'  # A step cannot proceed until some action is completed
@@ -28,11 +33,18 @@
 )
 
 STEP_FINAL_STATUSES = (STEP_STATUS_SKIPPED, STEP_STATUS_FAILED, STEP_STATUS_SUCCESS)
+STEP_EXECUTED_STATUSES = (STEP_STATUS_FAILED, STEP_STATUS_SUCCESS)
 
-RELEASE_STATUS_OPEN = 'open'
-RELEASE_STATUS_CLOSED = 'closed'
+
+RELEASE_STATUS_IN_PROGRESS = 'in progress'
+RELEASE_STATUS_FAILED = 'failed'
+RELEASE_STATUS_SUCCESS = 'success'
 
 RELEASE_STATUS_CHOICES = (
-    (RELEASE_STATUS_OPEN, RELEASE_STATUS_OPEN),
-    (RELEASE_STATUS_CLOSED, RELEASE_STATUS_CLOSED),
+    (RELEASE_STATUS_IN_PROGRESS, RELEASE_STATUS_IN_PROGRESS),
+    (RELEASE_STATUS_FAILED, RELEASE_STATUS_FAILED),
+    (RELEASE_STATUS_SUCCESS, RELEASE_STATUS_SUCCESS),
 )
+
+TAG_PRODUCTION_CHANGE_STARTED = "production_change_start"
+TAG_PRODUCTION_CHANGE_ENDED = "production_change_end"

katka/migrations/0022_change_release_status_fields.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.6 on 2019-10-21 07:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('katka', '0021_added_step_tags'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='scmrelease',
+            name='status',
+            field=models.CharField(choices=[('in progress', 'in progress'), ('failed', 'failed'), ('success', 'success')], default='in progress', max_length=30),
+        ),
+    ]

katka/models.py

@@ -6,7 +6,7 @@
 from encrypted_model_fields.fields import EncryptedCharField
 from katka.auditedmodel import AuditedModel
 from katka.constants import (
-    PIPELINE_STATUS_CHOICES, PIPELINE_STATUS_INITIALIZING, RELEASE_STATUS_CHOICES, RELEASE_STATUS_OPEN,
+    PIPELINE_STATUS_CHOICES, PIPELINE_STATUS_INITIALIZING, RELEASE_STATUS_CHOICES, RELEASE_STATUS_IN_PROGRESS,
     STEP_STATUS_CHOICES, STEP_STATUS_NOT_STARTED,
 )
 from katka.fields import KatkaSlugField
@@ -149,7 +149,7 @@ class Meta:
 
     public_identifier = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=255)
-    status = models.CharField(max_length=30, choices=RELEASE_STATUS_CHOICES, default=RELEASE_STATUS_OPEN)
+    status = models.CharField(max_length=30, choices=RELEASE_STATUS_CHOICES, default=RELEASE_STATUS_IN_PROGRESS)
     released = models.DateTimeField(null=True)
     from_hash = models.CharField(max_length=64)
     to_hash = models.CharField(max_length=64)

katka/releases.py

@@ -0,0 +1,95 @@
+import logging
+from dataclasses import dataclass
+
+from katka import constants
+from katka.fields import username_on_model
+from katka.models import SCMPipelineRun, SCMRelease, SCMStepRun
+
+log = logging.getLogger('katka')
+
+
+@dataclass
+class StepsPreConditions:
+    start_tag_executed: bool
+    end_tag_executed: bool
+    success_status_between_start_end: list
+
+    @property
+    def all_status_between_start_and_finish_are_success(self):
+        return all(self.success_status_between_start_end)
+
+    @property
+    def start_and_finish_tags_were_executed(self):
+        return self.start_tag_executed and self.end_tag_executed
+
+
+def create_release_if_necessary(pipeline):
+    release = _get_current_release(pipeline)
+    with username_on_model(SCMRelease, pipeline.modified_username):
+        if release is None:
+            release = SCMRelease.objects.create()
+
+        release.scm_pipeline_runs.add(pipeline)
+        release.save()
+
+
+def close_release_if_pipeline_finished(pipeline: SCMPipelineRun):
+    if pipeline.status not in constants.PIPELINE_FINAL_STATUSES:
+        log.debug(f'Pipeline {pipeline.public_identifier} not finished, doing nothing')
+        return None
+
+    pre_conditions = _gather_steps_pre_conditions(pipeline)
+    if not pre_conditions.start_and_finish_tags_were_executed:
+        log.debug(f'Pipeline {pipeline.public_identifier} missing start and/or finish tags')
+        return None
+
+    release = _get_current_release(pipeline)
+    if release and pre_conditions.all_status_between_start_and_finish_are_success:
+        release.status = constants.RELEASE_STATUS_SUCCESS
+    elif release:
+        log.debug(f'Pipeline {pipeline.public_identifier} contains step(s) with failed status')
+        release.status = constants.RELEASE_STATUS_FAILED
+    else:
+        log.warning(f'No open release for pipeline {pipeline.public_identifier}')
+
+    with username_on_model(SCMRelease, pipeline.modified_username):
+        if release:
+            release.save()
+
+    return release.status if release else None
+
+
+def _gather_steps_pre_conditions(pipeline):
+    steps = SCMStepRun.objects.filter(scm_pipeline_run=pipeline).order_by("sequence_id")
+    is_start_tag_executed = False
+    is_end_tag_executed = False
+    success_status_between_start_end = []
+    for step in steps:
+        if step.status not in constants.STEP_EXECUTED_STATUSES:
+            continue
+        if constants.TAG_PRODUCTION_CHANGE_STARTED in step.tags.split(" "):
+            is_start_tag_executed = True
+
+        if is_start_tag_executed:
+            success_status_between_start_end.append(step.status == constants.STEP_STATUS_SUCCESS)
+
+        if constants.TAG_PRODUCTION_CHANGE_ENDED in step.tags.split(" "):
+            is_end_tag_executed = True
+            break
+
+    return StepsPreConditions(is_start_tag_executed, is_end_tag_executed, success_status_between_start_end)
+
+
+def _get_current_release(pipeline):
+    releases = SCMRelease.objects.filter(
+        status=constants.RELEASE_STATUS_IN_PROGRESS, scm_pipeline_runs__application=pipeline.application
+    )
+    release = None
+    if len(releases) == 0:
+        log.error(f'No open releases found for application {pipeline.application.pk}')
+    elif len(releases) > 1:
+        log.error(f'Multiple open releases found for application {pipeline.application.pk}, picking newest')
+        release = releases.order_by('-created').first()
+    else:
+        release = releases[0]
+    return release

katka/signals.py

@@ -4,9 +4,10 @@
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
-from .constants import PIPELINE_STATUS_INITIALIZING, RELEASE_STATUS_OPEN, STEP_FINAL_STATUSES
-from .fields import username_on_model
-from .models import SCMPipelineRun, SCMRelease, SCMStepRun
+from katka.constants import PIPELINE_STATUS_INITIALIZING, STEP_FINAL_STATUSES
+from katka.fields import username_on_model
+from katka.models import SCMPipelineRun, SCMStepRun
+from katka.releases import close_release_if_pipeline_finished, create_release_if_necessary
 
 log = logging.getLogger('katka')
 
@@ -35,6 +36,8 @@ def send_pipeline_change_notification(sender, **kwargs):
     pipeline = kwargs['instance']
     if kwargs['created'] is True:
         create_release_if_necessary(pipeline)
+    else:
+        close_release_if_pipeline_finished(pipeline)
 
     if pipeline.status == PIPELINE_STATUS_INITIALIZING and kwargs['created'] is False:
         # Do not send notifications when the pipeline is initializing. While initializing, steps are created and
@@ -48,20 +51,3 @@ def send_pipeline_change_notification(sender, **kwargs):
     session.post(
         settings.PIPELINE_CHANGE_NOTIFICATION_URL, json={'public_identifier': str(pipeline.public_identifier)}
     )
-
-
-def create_release_if_necessary(pipeline):
-    releases = SCMRelease.objects.filter(
-        status=RELEASE_STATUS_OPEN, scm_pipeline_runs__application=pipeline.application
-    )
-    with username_on_model(SCMRelease, pipeline.modified_username):
-        if len(releases) == 0:
-            release = SCMRelease.objects.create()
-        elif len(releases) > 1:
-            log.error(f'Multiple open releases found for application {pipeline.application.pk}, picking newest')
-            release = releases.order_by('-created').first()
-        else:
-            release = releases[0]
-
-        release.scm_pipeline_runs.add(pipeline)
-        release.save()

tests/integration/conftest.py → tests/conftest.py

@@ -447,3 +447,109 @@ def not_my_metadata(not_my_application):
         meta.save()
 
     return meta
+
+
+@pytest.fixture
+def step_data(scm_pipeline_run):
+
+    steps = [
+        {"name": "step0", "stage": "prepare", "seq": "1.1-1", "status": "success", "tags": ""},
+        {"name": "step1", "stage": "prepare", "seq": "1.2-1", "status": "success", "tags": ""},
+        {"name": "step2", "stage": "deploy", "seq": "2.1-1", "status": "success", "tags": ""},
+        {"name": "step3", "stage": "deploy", "seq": "2.2-1", "status": "success", "tags": ""},
+        {"name": "step4", "stage": "deploy", "seq": "2.3-1", "status": "success", "tags": ""},
+        {"name": "step5", "stage": "deploy", "seq": "2.4-1", "status": "success", "tags": ""},
+        {"name": "step6", "stage": "deploy", "seq": "2.5-1", "status": "success", "tags": ""},
+    ]
+    return steps
+
+
+def _create_steps_from_dict(scm_pipeline_run, step_data):
+    saved_steps = []
+    for step in step_data:
+        scm_step_run = models.SCMStepRun(slug=step["name"], name=step["name"], stage=step["stage"],
+                                         scm_pipeline_run=scm_pipeline_run, sequence_id=step["seq"],
+                                         status=step["status"], tags=step["tags"])
+
+        with username_on_model(models.SCMStepRun, 'initial'):
+            scm_step_run.save()
+            saved_steps.append(scm_step_run)
+
+    return saved_steps
+
+
+@pytest.fixture
+def scm_step_run_success_list(scm_pipeline_run, step_data):
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_step_run_success_list_with_start_tag(scm_pipeline_run, step_data):
+    step_data[2]["tags"] = "production_change_start"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_step_run_success_list_with_start_end_tags(scm_pipeline_run, step_data):
+    step_data[2]["tags"] = "production_change_start"
+    step_data[5]["tags"] = "production_change_end"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_step_run_one_failed_step_list_with_start_end_tags(scm_pipeline_run, step_data):
+    step_data[2]["tags"] = "production_change_start"
+    step_data[4]["status"] = "failed"
+    step_data[5]["tags"] = "production_change_end"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_step_run_one_failed_step_before_start_tag(scm_pipeline_run, step_data):
+    step_data[1]["status"] = "failed"
+    step_data[2]["status"] = "skipped"
+    step_data[3]["status"] = "skipped"
+    step_data[4]["status"] = "skipped"
+    step_data[6]["status"] = "skipped"
+    step_data[2]["tags"] = "production_change_start"
+    step_data[5]["tags"] = "production_change_end"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_step_run_one_failed_step_after_end_tag(scm_pipeline_run, step_data):
+    step_data[2]["tags"] = "production_change_start"
+    step_data[5]["tags"] = "production_change_end"
+    step_data[6]["status"] = "failed"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_pipeline_run_with_no_open_release(another_project, another_scm_repository,
+                                          scm_step_run_success_list_with_start_end_tags):
+    application = models.Application(project=another_project,
+                                     scm_repository=another_scm_repository,
+                                     name='Application 2', slug='APP2')
+    with username_on_model(models.Application, 'initial'):
+        application.save()
+
+    pipeline_yaml = '''stages:
+             - release
+
+           do-release:
+             stage: release
+           '''
+    scm_pipeline_run = models.SCMPipelineRun(application=application,
+                                             pipeline_yaml=pipeline_yaml,
+                                             steps_total=5,
+                                             status="success",
+                                             commit_hash='4015B57A143AEC5156FD1444A017A32137A3FD0F')
+    with username_on_model(models.SCMPipelineRun, 'initial'):
+        scm_pipeline_run.save()
+
+    with username_on_model(models.SCMStepRun, 'initial'):
+        for step in scm_step_run_success_list_with_start_end_tags:
+            step.scm_pipeline_run = scm_pipeline_run
+            step.save()
+
+    return scm_pipeline_run

tests/integration/test_scmrelease_view.py

@@ -64,7 +64,7 @@ def test_list(self, client, logged_in_user, scm_pipeline_run, scm_release):
         assert parsed[0]['released'] is None
         assert parsed[0]['from_hash'] == '577fe3f6a091aa4bad996623b1548b87f4f9c1f8'
         assert parsed[0]['to_hash'] == 'a49954f060b1b7605e972c9448a74d4067547443'
-        assert parsed[0]['status'] == 'open'
+        assert parsed[0]['status'] == 'in progress'
         assert len(parsed[0]['scm_pipeline_runs']) == 1
         assert UUID(parsed[0]['scm_pipeline_runs'][0]) == scm_pipeline_run.public_identifier
         UUID(parsed[0]['public_identifier'])  # should not raise
@@ -81,7 +81,7 @@ def test_filtered_list(self, client, logged_in_user, scm_pipeline_run, scm_relea
         assert parsed[0]['released'] is None
         assert parsed[0]['from_hash'] == '100763d7144e1f993289bd528dc698dd3906a807'
         assert parsed[0]['to_hash'] == '38d72050370e6e0b43df649c9630f7135ef6de0d'
-        assert parsed[0]['status'] == 'open'
+        assert parsed[0]['status'] == 'in progress'
         assert len(parsed[0]['scm_pipeline_runs']) == 1
         assert UUID(parsed[0]['scm_pipeline_runs'][0]) == another_scm_pipeline_run.public_identifier
 
@@ -107,7 +107,7 @@ def test_get(self, client, logged_in_user, scm_pipeline_run, scm_release):
         assert parsed['released'] is None
         assert parsed['from_hash'] == '577fe3f6a091aa4bad996623b1548b87f4f9c1f8'
         assert parsed['to_hash'] == 'a49954f060b1b7605e972c9448a74d4067547443'
-        assert parsed['status'] == 'open'
+        assert parsed['status'] == 'in progress'
         assert len(parsed['scm_pipeline_runs']) == 1
         assert UUID(parsed['scm_pipeline_runs'][0]) == scm_pipeline_run.public_identifier