Merge pull request #43 from rodgomes/master

BREAK: fill in extra fields when closing release

katka/admin.py

@@ -71,5 +71,5 @@ class ApplicationMetadataAdmin(WithUsernameAdminModel):
 
 @admin.register(SCMRelease)
 class SCMReleaseAdmin(WithUsernameAdminModel):
-    fields = ('name', 'status', 'released', 'from_hash', 'to_hash', 'scm_pipeline_runs')
+    fields = ('name', 'status', 'started', 'ended', 'scm_pipeline_runs')
     list_display = ('pk', 'name')

katka/migrations/0023_auto_20191024_0825.py

@@ -0,0 +1,35 @@
+# Generated by Django 2.2.6 on 2019-10-24 08:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('katka', '0022_change_release_status_fields'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='scmrelease',
+            name='from_hash',
+        ),
+        migrations.RemoveField(
+            model_name='scmrelease',
+            name='released',
+        ),
+        migrations.RemoveField(
+            model_name='scmrelease',
+            name='to_hash',
+        ),
+        migrations.AddField(
+            model_name='scmrelease',
+            name='ended',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='scmrelease',
+            name='started',
+            field=models.DateTimeField(null=True),
+        ),
+    ]

katka/models.py

@@ -150,9 +150,8 @@ class Meta:
     public_identifier = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=255)
     status = models.CharField(max_length=30, choices=RELEASE_STATUS_CHOICES, default=RELEASE_STATUS_IN_PROGRESS)
-    released = models.DateTimeField(null=True)
-    from_hash = models.CharField(max_length=64)
-    to_hash = models.CharField(max_length=64)
+    started = models.DateTimeField(null=True)
+    ended = models.DateTimeField(null=True)
     scm_pipeline_runs = models.ManyToManyField(SCMPipelineRun)
 
 

katka/releases.py

@@ -1,5 +1,7 @@
+import json
 import logging
 from dataclasses import dataclass
+from datetime import datetime
 
 from katka import constants
 from katka.fields import username_on_model
@@ -10,17 +12,18 @@
 
 @dataclass
 class StepsPreConditions:
-    start_tag_executed: bool
-    end_tag_executed: bool
+    version_number: str
     success_status_between_start_end: list
+    prod_change_start_date: datetime = None
+    prod_change_end_date: datetime = None
 
     @property
     def all_status_between_start_and_finish_are_success(self):
         return all(self.success_status_between_start_end)
 
     @property
     def start_and_finish_tags_were_executed(self):
-        return self.start_tag_executed and self.end_tag_executed
+        return self.prod_change_start_date is not None and self.prod_change_end_date is not None
 
 
 def create_release_if_necessary(pipeline):
@@ -43,6 +46,10 @@ def close_release_if_pipeline_finished(pipeline: SCMPipelineRun):
         log.debug(f'Pipeline {pipeline.public_identifier} missing start and/or finish tags')
         return None
 
+    if not pre_conditions.version_number:
+        log.debug(f'Pipeline {pipeline.public_identifier} missing release version')
+        return None
+
     release = _get_current_release(pipeline)
     if release and pre_conditions.all_status_between_start_and_finish_are_success:
         release.status = constants.RELEASE_STATUS_SUCCESS
@@ -54,30 +61,47 @@ def close_release_if_pipeline_finished(pipeline: SCMPipelineRun):
 
     with username_on_model(SCMRelease, pipeline.modified_username):
         if release:
+            release.name = pre_conditions.version_number
+            release.started = pre_conditions.prod_change_start_date
+            release.ended = pre_conditions.prod_change_end_date
             release.save()
 
     return release.status if release else None
 
 
 def _gather_steps_pre_conditions(pipeline):
     steps = SCMStepRun.objects.filter(scm_pipeline_run=pipeline).order_by("sequence_id")
-    is_start_tag_executed = False
-    is_end_tag_executed = False
+    prod_start_date = None
+    prod_end_date = None
     success_status_between_start_end = []
+    pipeline_output = {}
     for step in steps:
         if step.status not in constants.STEP_EXECUTED_STATUSES:
             continue
+        _add_output(pipeline_output=pipeline_output, step_output=step.output)
         if constants.TAG_PRODUCTION_CHANGE_STARTED in step.tags.split(" "):
-            is_start_tag_executed = True
+            prod_start_date = step.modified
 
-        if is_start_tag_executed:
+        if prod_start_date is not None:
             success_status_between_start_end.append(step.status == constants.STEP_STATUS_SUCCESS)
 
         if constants.TAG_PRODUCTION_CHANGE_ENDED in step.tags.split(" "):
-            is_end_tag_executed = True
+            prod_end_date = step.modified
             break
 
-    return StepsPreConditions(is_start_tag_executed, is_end_tag_executed, success_status_between_start_end)
+    return StepsPreConditions(pipeline_output.get("release.version"),
+                              success_status_between_start_end,
+                              prod_change_start_date=prod_start_date,
+                              prod_change_end_date=prod_end_date)
+
+
+def _add_output(pipeline_output: dict, step_output: str) -> None:
+    if not step_output:
+        return
+    try:
+        pipeline_output.update(json.loads(step_output))
+    except json.JSONDecodeError:
+        logging.warning("Invalid JSON in step output")
 
 
 def _get_current_release(pipeline):

katka/serializers.py

@@ -129,22 +129,18 @@ def __init__(self, *args, **kwargs):
 class SCMReleaseSerializer(KatkaSerializer):
     scm_pipeline_runs = SCMPipelineRunRelatedField(required=False, read_only=True, many=True)
 
-    released = serializers.DateTimeField(required=False)
-
     class Meta:
         model = SCMRelease
-        fields = ('public_identifier', 'name', 'released', 'from_hash', 'to_hash', 'scm_pipeline_runs', 'status')
-        read_only_fields = ('from_hash', 'to_hash', 'scm_pipeline_runs')
+        fields = ('public_identifier', 'name', 'started', 'ended', 'scm_pipeline_runs', 'status')
+        read_only_fields = ('started', 'ended', 'scm_pipeline_runs')
 
 
 class SCMReleaseCreateSerializer(KatkaSerializer):
     scm_pipeline_runs = SCMPipelineRunRelatedField(required=False, many=True)
 
-    released = serializers.DateTimeField(required=False)
-
     class Meta:
         model = SCMRelease
-        fields = ('public_identifier', 'name', 'released', 'from_hash', 'to_hash', 'scm_pipeline_runs', 'status')
+        fields = ('name', 'scm_pipeline_runs', 'status')
 
 
 class ApplicationMetadataSerializer(serializers.ModelSerializer):

tests/conftest.py

@@ -382,8 +382,6 @@ def scm_release(scm_pipeline_run):
 
     with username_on_model(models.SCMRelease, 'initial'):
         scm_release.name = 'Version 0.13.1'
-        scm_release.from_hash = '577fe3f6a091aa4bad996623b1548b87f4f9c1f8'
-        scm_release.to_hash = 'a49954f060b1b7605e972c9448a74d4067547443'
         scm_release.save()
 
     return scm_release
@@ -404,8 +402,6 @@ def another_scm_release(another_scm_pipeline_run):
 
     with username_on_model(models.SCMRelease, 'initial'):
         scm_release.name = 'Version 15.0'
-        scm_release.from_hash = '100763d7144e1f993289bd528dc698dd3906a807'
-        scm_release.to_hash = '38d72050370e6e0b43df649c9630f7135ef6de0d'
         scm_release.save()
 
     return scm_release
@@ -451,15 +447,15 @@ def not_my_metadata(not_my_application):
 
 @pytest.fixture
 def step_data(scm_pipeline_run):
-
+    version = '{"release.version": "1.0.0"}'
     steps = [
-        {"name": "step0", "stage": "prepare", "seq": "1.1-1", "status": "success", "tags": ""},
-        {"name": "step1", "stage": "prepare", "seq": "1.2-1", "status": "success", "tags": ""},
-        {"name": "step2", "stage": "deploy", "seq": "2.1-1", "status": "success", "tags": ""},
-        {"name": "step3", "stage": "deploy", "seq": "2.2-1", "status": "success", "tags": ""},
-        {"name": "step4", "stage": "deploy", "seq": "2.3-1", "status": "success", "tags": ""},
-        {"name": "step5", "stage": "deploy", "seq": "2.4-1", "status": "success", "tags": ""},
-        {"name": "step6", "stage": "deploy", "seq": "2.5-1", "status": "success", "tags": ""},
+        {"name": "step0", "stage": "prepare", "seq": "1.1-1", "status": "success", "tags": "", "output": version},
+        {"name": "step1", "stage": "prepare", "seq": "1.2-1", "status": "success", "tags": "", "output": ""},
+        {"name": "step2", "stage": "deploy", "seq": "2.1-1", "status": "success", "tags": "", "output": ""},
+        {"name": "step3", "stage": "deploy", "seq": "2.2-1", "status": "success", "tags": "", "output": ""},
+        {"name": "step4", "stage": "deploy", "seq": "2.3-1", "status": "success", "tags": "", "output": ""},
+        {"name": "step5", "stage": "deploy", "seq": "2.4-1", "status": "success", "tags": "", "output": ""},
+        {"name": "step6", "stage": "deploy", "seq": "2.5-1", "status": "success", "tags": "", "output": ""},
     ]
     return steps
 
@@ -469,7 +465,7 @@ def _create_steps_from_dict(scm_pipeline_run, step_data):
     for step in step_data:
         scm_step_run = models.SCMStepRun(slug=step["name"], name=step["name"], stage=step["stage"],
                                          scm_pipeline_run=scm_pipeline_run, sequence_id=step["seq"],
-                                         status=step["status"], tags=step["tags"])
+                                         status=step["status"], tags=step["tags"], output=step["output"])
 
         with username_on_model(models.SCMStepRun, 'initial'):
             scm_step_run.save()
@@ -524,6 +520,22 @@ def scm_step_run_one_failed_step_after_end_tag(scm_pipeline_run, step_data):
     return _create_steps_from_dict(scm_pipeline_run, step_data)
 
 
+@pytest.fixture
+def scm_step_run_without_version_output(scm_pipeline_run, step_data):
+    step_data[0]["output"] = ""
+    step_data[2]["tags"] = "production_change_start"
+    step_data[5]["tags"] = "production_change_end"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
+@pytest.fixture
+def scm_step_run_with_broken_output(scm_pipeline_run, step_data):
+    step_data[2]["output"] = "1:2"
+    step_data[2]["tags"] = "production_change_start"
+    step_data[5]["tags"] = "production_change_end"
+    return _create_steps_from_dict(scm_pipeline_run, step_data)
+
+
 @pytest.fixture
 def scm_pipeline_run_with_no_open_release(another_project, another_scm_repository,
                                           scm_step_run_success_list_with_start_end_tags):

tests/integration/test_scmrelease_view.py

@@ -30,8 +30,6 @@ def test_delete(self, client, scm_release):
     def test_update(self, client, scm_pipeline_run, scm_release):
         url = f'/scm-releases/{scm_release.public_identifier}/'
         data = {'name': 'Version 1', 'released': '2018-06-16 12:34:56',
-                'from_hash': '4015B57A143AEC5156FD1444A017A32137A3FD0F',
-                'to_hash': '0a032e92f77797d9be0ea3ad6c595392313ded72',
                 'scm_pipeline_run': scm_pipeline_run.public_identifier}
         response = client.put(url, data, content_type='application/json')
         assert response.status_code == 405
@@ -45,8 +43,6 @@ def test_partial_update(self, client, scm_release):
     def test_create(self, client, scm_pipeline_run, scm_release):
         url = f'/scm-releases/'
         data = {'name': 'Version 1',
-                'from_hash': '4015B57A143AEC5156FD1444A017A32137A3FD0F',
-                'to_hash': '0a032e92f77797d9be0ea3ad6c595392313ded72',
                 'scm_pipeline_runs': [scm_pipeline_run.public_identifier]}
         response = client.post(url, data=data, content_type='application/json')
         assert response.status_code == 405
@@ -61,9 +57,8 @@ def test_list(self, client, logged_in_user, scm_pipeline_run, scm_release):
         parsed = response.json()
         assert len(parsed) == 1
         assert parsed[0]['name'] == 'Version 0.13.1'
-        assert parsed[0]['released'] is None
-        assert parsed[0]['from_hash'] == '577fe3f6a091aa4bad996623b1548b87f4f9c1f8'
-        assert parsed[0]['to_hash'] == 'a49954f060b1b7605e972c9448a74d4067547443'
+        assert parsed[0]['started'] is None
+        assert parsed[0]['ended'] is None
         assert parsed[0]['status'] == 'in progress'
         assert len(parsed[0]['scm_pipeline_runs']) == 1
         assert UUID(parsed[0]['scm_pipeline_runs'][0]) == scm_pipeline_run.public_identifier
@@ -78,9 +73,8 @@ def test_filtered_list(self, client, logged_in_user, scm_pipeline_run, scm_relea
         parsed = response.json()
         assert len(parsed) == 1
         assert parsed[0]['name'] == 'Version 15.0'
-        assert parsed[0]['released'] is None
-        assert parsed[0]['from_hash'] == '100763d7144e1f993289bd528dc698dd3906a807'
-        assert parsed[0]['to_hash'] == '38d72050370e6e0b43df649c9630f7135ef6de0d'
+        assert parsed[0]['started'] is None
+        assert parsed[0]['ended'] is None
         assert parsed[0]['status'] == 'in progress'
         assert len(parsed[0]['scm_pipeline_runs']) == 1
         assert UUID(parsed[0]['scm_pipeline_runs'][0]) == another_scm_pipeline_run.public_identifier
@@ -104,9 +98,8 @@ def test_get(self, client, logged_in_user, scm_pipeline_run, scm_release):
         assert response.status_code == 200
         parsed = response.json()
         assert parsed['name'] == 'Version 0.13.1'
-        assert parsed['released'] is None
-        assert parsed['from_hash'] == '577fe3f6a091aa4bad996623b1548b87f4f9c1f8'
-        assert parsed['to_hash'] == 'a49954f060b1b7605e972c9448a74d4067547443'
+        assert parsed['started'] is None
+        assert parsed['ended'] is None
         assert parsed['status'] == 'in progress'
         assert len(parsed['scm_pipeline_runs']) == 1
         assert UUID(parsed['scm_pipeline_runs'][0]) == scm_pipeline_run.public_identifier
@@ -121,33 +114,15 @@ def test_delete(self, client, logged_in_user, scm_release):
 
     def test_update(self, client, logged_in_user, scm_pipeline_run, scm_release):
         url = f'/scm-releases/{scm_release.public_identifier}/'
-        data = {'name': 'Version 1', 'released': '2018-06-16 12:34:56',
-                'from_hash': '4015B57A143AEC5156FD1444A017A32137A3FD0F',
-                'to_hash': '0a032e92f77797d9be0ea3ad6c595392313ded72',
-                'scm_pipeline_run': scm_pipeline_run.public_identifier}
+        data = {'name': 'Version 1', 'scm_pipeline_run': scm_pipeline_run.public_identifier}
         response = client.put(url, data, content_type='application/json')
         assert response.status_code == 405
         p = models.SCMRelease.objects.get(pk=scm_release.public_identifier)
         assert p.name == 'Version 0.13.1'
 
-    def test_update_cannot_change_hashes(self, client, logged_in_user, scm_pipeline_run, scm_release):
-        url = f'/scm-releases/{scm_release.public_identifier}/'
-        data = {'name': 'Version hash', 'released': '2018-06-16 12:34:56',
-                'from_hash': 'BBBBB57A143AEC5156FD1444A017A32137A3FD0F',
-                'to_hash': 'AAAAe92f77797d9be0ea3ad6c595392313ded72',
-                'scm_pipeline_run': scm_pipeline_run.public_identifier}
-        response = client.put(url, data, content_type='application/json')
-        assert response.status_code == 405
-        p = models.SCMRelease.objects.get(pk=scm_release.public_identifier)
-        assert p.from_hash != 'BBBBB57A143AEC5156FD1444A017A32137A3FD0F'
-        assert p.to_hash != 'AAAAe92f77797d9be0ea3ad6c595392313ded72'
-
     def test_update_cannot_change_pipeline_run(self, client, logged_in_user, another_scm_pipeline_run, scm_release):
         url = f'/scm-releases/{scm_release.public_identifier}/'
-        data = {'name': 'Version pipeline run', 'released': '2018-06-16 12:34:56',
-                'from_hash': '577fe3f6a091aa4bad996623b1548b87f4f9c1f8',
-                'to_hash': 'a49954f060b1b7605e972c9448a74d4067547443',
-                'scm_pipeline_runs': [another_scm_pipeline_run.public_identifier]}
+        data = {'name': 'Version pipeline run', 'scm_pipeline_runs': [another_scm_pipeline_run.public_identifier]}
         response = client.put(url, data, content_type='application/json')
         assert response.status_code == 405
         p = models.SCMRelease.objects.get(pk=scm_release.public_identifier)
@@ -167,8 +142,6 @@ def test_partial_update(self, client, logged_in_user, scm_release):
     def test_create(self, client, logged_in_user, scm_pipeline_run):
         url = f'/scm-releases/'
         data = {'name': 'Version create',
-                'from_hash': '4015B57A143AEC5156FD1444A017A32137A3FD0F',
-                'to_hash': '0a032e92f77797d9be0ea3ad6c595392313ded72',
                 'scm_pipeline_runs': f'{scm_pipeline_run.public_identifier},'}
         response = client.post(url, data=data, content_type='application/json')
         assert response.status_code == 405

tests/unit/test_close_release.py

@@ -7,6 +7,15 @@
 @pytest.mark.django_db
 class TestCloseRelease:
 
+    @staticmethod
+    def _assert_release_success_with_name(name):
+        assert SCMRelease.objects.count() == 1
+        release = SCMRelease.objects.first()
+        assert release.name == name
+        assert release.status == constants.RELEASE_STATUS_SUCCESS
+        assert release.started is not None
+        assert release.ended is not None
+
     @staticmethod
     def _assert_release_has_status(status):
         assert SCMRelease.objects.count() == 1
@@ -55,7 +64,7 @@ def test_all_steps_are_success_with_start_and_end_tag(self, scm_pipeline_run,
         result_status = close_release_if_pipeline_finished(scm_pipeline_run)
         assert result_status == constants.PIPELINE_STATUS_SUCCESS
 
-        self._assert_release_has_status(constants.RELEASE_STATUS_SUCCESS)
+        self._assert_release_success_with_name("1.0.0")
 
     def test_one_failed_step_between_start_end_tags(self, scm_pipeline_run,
                                                     scm_step_run_one_failed_step_list_with_start_end_tags):
@@ -84,5 +93,24 @@ def test_one_failed_step_after_end_tag(self, scm_pipeline_run,
         scm_pipeline_run.status = constants.PIPELINE_STATUS_FAILED
         result_status = close_release_if_pipeline_finished(scm_pipeline_run)
         assert result_status == constants.RELEASE_STATUS_SUCCESS
+        self._assert_release_success_with_name("1.0.0")
+
+    def test_fails_if_version_not_present_in_context(self, scm_pipeline_run,
+                                                     scm_step_run_without_version_output):
+        self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS)
+
+        scm_pipeline_run.status = constants.PIPELINE_STATUS_SUCCESS
+        result_status = close_release_if_pipeline_finished(scm_pipeline_run)
+        assert result_status is None
+
+        self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS)
+
+    def test_output_with_broken_json_does_not_break_release(self, scm_pipeline_run,
+                                                            scm_step_run_with_broken_output):
+        self._assert_release_has_status(constants.RELEASE_STATUS_IN_PROGRESS)
+
+        scm_pipeline_run.status = constants.PIPELINE_STATUS_SUCCESS
+        result_status = close_release_if_pipeline_finished(scm_pipeline_run)
+        assert result_status == constants.PIPELINE_STATUS_SUCCESS
 
-        self._assert_release_has_status(constants.RELEASE_STATUS_SUCCESS)
+        self._assert_release_success_with_name("1.0.0")