From 8724407769bf5304c5fdd66ec4e95a4dbd5c7981 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Wed, 21 Aug 2024 13:08:32 +0200 Subject: [PATCH 1/7] airflow wf: close curation not working fix --- .github/workflows/test-workflows.yml | 4 +- docker-compose.yaml | 2 +- .../author_create/author_create_approved.py | 12 +-- .../author_create/author_create_rejected.py | 17 +--- .../dags/author/author_create/shared_tasks.py | 17 ++++ workflows/requirements-test.txt | 2 + workflows/tests/__init__.py | 0 ....test_close_author_create_user_ticket.yaml | 90 +++++++++++++++++++ workflows/tests/test_author_create_tasks.py | 17 ++++ 9 files changed, 131 insertions(+), 30 deletions(-) create mode 100644 workflows/dags/author/author_create/shared_tasks.py create mode 100644 workflows/tests/__init__.py create mode 100644 workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml create mode 100644 workflows/tests/test_author_create_tasks.py diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index cebc3667a..4d1824b17 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -51,7 +51,7 @@ jobs: -v "$(pwd)"/tests:/opt/airflow/tests -v "$(pwd)"/requirements-test.txt:/opt/airflow/requirements-test.txt -v "$(pwd)"/data:/opt/airflow/data - -v "$(pwd)"/scripts/variables/variables.json:/opt/airflow/variables.json + -v "$(pwd)"/scripts:/opt/airflow/scripts -e AIRFLOW__CORE__EXECUTOR=CeleryExecutor -e AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@127.0.0.1:5432/airflow -e AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@127.0.0.1:5432/airflow @@ -61,4 +61,4 @@ jobs: -e AIRFLOW__CORE__LOAD_EXAMPLES="false" -e AIRFLOW__API__AUTH_BACKENDS="airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" registry.cern.ch/cern-sis/inspire/workflows@${{ needs.build.outputs.image-id }} - bash -c "pip install -r requirements-test.txt && airflow db init && airflow variables import /opt/airflow/variables.json && pytest /opt/airflow/tests" + bash -c "pip install -r requirements-test.txt && airflow db init && airflow connections import /opt/airflow/scripts/connections/connections.json && airflow variables import /opt/airflow/scripts/variables/variables.json && pytest /opt/airflow/tests" diff --git a/docker-compose.yaml b/docker-compose.yaml index 96b5111d0..3f33162d1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -57,7 +57,7 @@ x-airflow-common: &airflow-common AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres-airflow/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 AIRFLOW__CORE__FERNET_KEY: "" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "false" AIRFLOW__CORE__LOAD_EXAMPLES: "false" AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__WEBSERVER__RELOAD_ON_PLUGIN_CHANGE: "true" # used when modifying plugins diff --git a/workflows/dags/author/author_create/author_create_approved.py b/workflows/dags/author/author_create/author_create_approved.py index 083ae9050..8927dd25a 100644 --- a/workflows/dags/author/author_create/author_create_approved.py +++ b/workflows/dags/author/author_create/author_create_approved.py @@ -3,7 +3,7 @@ from airflow.decorators import dag, task from airflow.models.param import Param -from airflow.utils.trigger_rule import TriggerRule +from author.author_create.shared_tasks import close_author_create_user_ticket from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook from hooks.backoffice.workflow_ticket_management_hook import ( WorkflowTicketManagementHook, @@ -119,16 +119,6 @@ def author_create_success_branch(**context: dict) -> str: else: return "set_author_create_workflow_status_to_error" - @task(trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS) - def close_author_create_user_ticket(**context: dict) -> None: - ticket_type = "author_create_user" - ticket_id = workflow_ticket_management_hook.get_ticket( - workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type - )["ticket_id"] - endpoint = "api/tickets/resolve" - request_data = {"ticket_id": ticket_id} - inspire_http_hook.call_api(endpoint=endpoint, data=request_data, method="POST") - @task() def set_author_create_workflow_status_to_completed(**context: dict) -> None: status_name = "completed" diff --git a/workflows/dags/author/author_create/author_create_rejected.py b/workflows/dags/author/author_create/author_create_rejected.py index e492038a9..9de52a1b8 100644 --- a/workflows/dags/author/author_create/author_create_rejected.py +++ b/workflows/dags/author/author_create/author_create_rejected.py @@ -2,11 +2,8 @@ from airflow.decorators import dag, task from airflow.models.param import Param +from author.author_create.shared_tasks import close_author_create_user_ticket from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook -from hooks.backoffice.workflow_ticket_management_hook import ( - WorkflowTicketManagementHook, -) -from hooks.inspirehep.inspire_http_hook import InspireHttpHook from include.utils.set_workflow_status import set_workflow_status_to_error @@ -31,19 +28,7 @@ def author_create_rejected_dag() -> None: 2. set_author_create_workflow_status_to_completed: Sets the status of the author creation workflow to 'completed'. """ - inspire_http_hook = InspireHttpHook() workflow_management_hook = WorkflowManagementHook() - workflow_ticket_management_hook = WorkflowTicketManagementHook() - - @task() - def close_author_create_user_ticket(**context: dict) -> None: - ticket_type = "author_create_user" - ticket_id = workflow_ticket_management_hook.get_ticket( - workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type - )["ticket_id"] - endpoint = "/tickets/resolve" # TODO: the URL for resolving dag will change - request_data = {"ticket_id": ticket_id} - inspire_http_hook.call_api(endpoint=endpoint, data=request_data, method="POST") @task() def set_author_create_workflow_status_to_completed(**context: dict) -> None: diff --git a/workflows/dags/author/author_create/shared_tasks.py b/workflows/dags/author/author_create/shared_tasks.py new file mode 100644 index 000000000..b9b988030 --- /dev/null +++ b/workflows/dags/author/author_create/shared_tasks.py @@ -0,0 +1,17 @@ +from airflow.decorators import task +from airflow.utils.trigger_rule import TriggerRule +from hooks.backoffice.workflow_ticket_management_hook import ( + WorkflowTicketManagementHook, +) +from hooks.inspirehep.inspire_http_hook import InspireHttpHook + + +@task(trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS) +def close_author_create_user_ticket(**context: dict) -> None: + ticket_type = "author_create_user" + ticket_id = WorkflowTicketManagementHook().get_ticket( + workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type + )["ticket_id"] + endpoint = "api/tickets/resolve" + request_data = {"ticket_id": ticket_id} + InspireHttpHook().call_api(endpoint=endpoint, data=request_data, method="POST") diff --git a/workflows/requirements-test.txt b/workflows/requirements-test.txt index 0c69b775f..de55b38d3 100644 --- a/workflows/requirements-test.txt +++ b/workflows/requirements-test.txt @@ -1,3 +1,5 @@ pytest coverage pytest-cov +pytest-vcr==1.0.2 +vcrpy==6.0.1 diff --git a/workflows/tests/__init__.py b/workflows/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml b/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml new file mode 100644 index 000000000..d7371e1c4 --- /dev/null +++ b/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml @@ -0,0 +1,90 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Authorization: + - Token 2e04111a61e8f5ba6ecec52af21bbb9e81732085 + Connection: + - keep-alive + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: GET + uri: http://host.docker.internal:8000/api/workflow-ticket/f8301c06-8fa1-4124-845e-c270b910af5f/?ticket_type=author_create_user + response: + body: + string: '{"id":1,"ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=656f2d17878c929095f833340cbb3531","ticket_id":"656f2d17878c929095f833340cbb3531","ticket_type":"author_create_user","workflow_id":"f8301c06-8fa1-4124-845e-c270b910af5f"}' + headers: + Allow: + - GET, HEAD, OPTIONS + Content-Language: + - en + Content-Length: + - '275' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Wed, 21 Aug 2024 11:06:38 GMT + Referrer-Policy: + - same-origin + Server: + - WSGIServer/0.2 CPython/3.11.6 + Vary: + - Accept, Accept-Language, Cookie, origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +- request: + body: ticket_id=656f2d17878c929095f833340cbb3531 + headers: + Accept: + - application/vnd+inspire.record.raw+json + Accept-Encoding: + - gzip, deflate + Authorization: + - Bearer cZiS4W7K8sqyebkxQzpnSwuUKLr5Ne6qPfnoOAjP7M2IvHxQhKmwiCJpp2QC + Connection: + - keep-alive + Content-Length: + - '42' + Content-Type: + - application/x-www-form-urlencoded + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://inspirebeta.net/api/tickets/resolve + response: + body: + string: '{"message":"Ticket resolved"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + content-length: + - '30' + content-type: + - application/json + date: + - Wed, 21 Aug 2024 11:06:39 GMT + server: + - gunicorn/19.10.0 + x-proxy-backend: + - inspire-qa_hep-web_http + status: + code: 200 + message: OK +version: 1 diff --git a/workflows/tests/test_author_create_tasks.py b/workflows/tests/test_author_create_tasks.py new file mode 100644 index 000000000..1958145a7 --- /dev/null +++ b/workflows/tests/test_author_create_tasks.py @@ -0,0 +1,17 @@ +import pytest +from dags.author.author_create.shared_tasks import ( + close_author_create_user_ticket, +) + + +class TestAuthorCreate: + context = { + "params": { + "workflow_id": "f8301c06-8fa1-4124-845e-c270b910af5f", + "data": {"value": "reject", "create_ticket": False}, + } + } + + @pytest.mark.vcr() + def test_close_author_create_user_ticket(self): + close_author_create_user_ticket.function(**self.context) From f658c3d617e7d4c9dad8eb2d75980b2627828044 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Tue, 20 Aug 2024 17:02:19 +0200 Subject: [PATCH 2/7] workflows: add decision table and populate it * Decision is now added on the /resolve api * ref: cern-sis/issues-inspire/issues/522 --- backoffice/backoffice/workflows/admin.py | 79 ++++++++++++------- .../backoffice/workflows/api/serializers.py | 10 ++- backoffice/backoffice/workflows/api/utils.py | 11 +++ backoffice/backoffice/workflows/api/views.py | 16 +++- backoffice/backoffice/workflows/constants.py | 4 + .../workflows/migrations/0009_decision.py | 58 ++++++++++++++ backoffice/backoffice/workflows/models.py | 13 +++ .../backoffice/workflows/tests/test_utils.py | 44 +++++++++++ .../backoffice/workflows/tests/test_views.py | 37 ++++++++- backoffice/config/api_router.py | 2 + 10 files changed, 240 insertions(+), 34 deletions(-) create mode 100644 backoffice/backoffice/workflows/api/utils.py create mode 100644 backoffice/backoffice/workflows/migrations/0009_decision.py create mode 100644 backoffice/backoffice/workflows/tests/test_utils.py diff --git a/backoffice/backoffice/workflows/admin.py b/backoffice/backoffice/workflows/admin.py index 308def4b6..3aa268277 100644 --- a/backoffice/backoffice/workflows/admin.py +++ b/backoffice/backoffice/workflows/admin.py @@ -3,7 +3,7 @@ from django_json_widget.widgets import JSONEditorWidget from backoffice.management.permissions import IsAdminOrCuratorUser -from backoffice.workflows.models import Workflow +from backoffice.workflows.models import Decision, Workflow class WorkflowsAdminSite(admin.AdminSite): @@ -30,8 +30,41 @@ def has_permission(self, request): ) +class BaseModelAdmin(admin.ModelAdmin): + def has_view_permission(self, request, obj=None): + """ + Returns True if the user has permission to view the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + def has_change_permission(self, request, obj=None): + """ + Returns True if the user has permission to change the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + def has_delete_permission(self, request, obj=None): + """ + Returns True if the user has permission to delete the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + formfield_overrides = { + JSONField: {"widget": JSONEditorWidget}, + } + + @admin.register(Workflow) -class WorkflowAdmin(admin.ModelAdmin): +class WorkflowAdmin(BaseModelAdmin): """ Admin class for Workflow model. Define get, update and delete permissions. """ @@ -56,33 +89,21 @@ class WorkflowAdmin(admin.ModelAdmin): "_updated_at", ] - formfield_overrides = { - JSONField: {"widget": JSONEditorWidget}, - } - def has_view_permission(self, request, obj=None): - """ - Returns True if the user has permission to view the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) +@admin.register(Decision) +class DecisionAdmin(BaseModelAdmin): + """ + Admin class for Decision model. Define get, update and delete permissions. + """ - def has_change_permission(self, request, obj=None): - """ - Returns True if the user has permission to change the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) + ordering = ("-_updated_at",) + search_fields = ["id", "data"] + list_display = ("id", "action_value", "user", "workflow_id") + list_filter = [ + "action", + "user", + ] - def has_delete_permission(self, request, obj=None): - """ - Returns True if the user has permission to delete the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) + @admin.display(description="action") + def action_value(self, obj): + return obj.action diff --git a/backoffice/backoffice/workflows/api/serializers.py b/backoffice/backoffice/workflows/api/serializers.py index 8452342d7..f4459d4ac 100644 --- a/backoffice/backoffice/workflows/api/serializers.py +++ b/backoffice/backoffice/workflows/api/serializers.py @@ -6,7 +6,7 @@ from backoffice.workflows.constants import ResolutionDags, StatusChoices, WorkflowType from backoffice.workflows.documents import WorkflowDocument -from backoffice.workflows.models import Workflow, WorkflowTicket +from backoffice.workflows.models import Decision, Workflow, WorkflowTicket class WorkflowTicketSerializer(serializers.ModelSerializer): @@ -31,6 +31,14 @@ class Meta: fields = "__all__" +class DecisionSerializer(serializers.ModelSerializer): + workflow = serializers.PrimaryKeyRelatedField(queryset=Workflow.objects.all()) + + class Meta: + model = Decision + fields = "__all__" + + class WorkflowDocumentSerializer(DocumentSerializer): class Meta: document = WorkflowDocument diff --git a/backoffice/backoffice/workflows/api/utils.py b/backoffice/backoffice/workflows/api/utils.py new file mode 100644 index 000000000..26868e981 --- /dev/null +++ b/backoffice/backoffice/workflows/api/utils.py @@ -0,0 +1,11 @@ +from backoffice.workflows.api.serializers import DecisionSerializer + + +def add_decision(workflow_id, user, action): + serializer_class = DecisionSerializer + data = {"workflow": workflow_id, "user": user, "action": action} + + serializer = serializer_class(data=data) + if serializer.is_valid(raise_exception=True): + serializer.save() + return serializer.data diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 42df6630f..eb4652885 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -23,6 +23,7 @@ from backoffice.utils.pagination import OSStandardResultsSetPagination from backoffice.workflows import airflow_utils +from backoffice.workflows.api import utils from backoffice.workflows.api.serializers import ( AuthorResolutionSerializer, WorkflowAuthorSerializer, @@ -37,7 +38,7 @@ WorkflowType, ) from backoffice.workflows.documents import WorkflowDocument -from backoffice.workflows.models import Workflow, WorkflowTicket +from backoffice.workflows.models import Decision, Workflow, WorkflowTicket logger = logging.getLogger(__name__) @@ -100,6 +101,16 @@ def create(self, request, *args, **kwargs): ) +class DecisionViewSet(viewsets.ModelViewSet): + queryset = Decision.objects.all() + + def create(self, request, *args, **kwargs): + data = utils.add_decision( + request.data["workflow_id"], request.user, request.data["action"] + ) + return Response(data, status=status.HTTP_201_CREATED) + + class AuthorWorkflowViewSet(viewsets.ViewSet): serializer_class = WorkflowAuthorSerializer @@ -160,12 +171,13 @@ def resolve(self, request, pk=None): logger.info("Resolving data: %s", request.data) serializer = AuthorResolutionSerializer(data=request.data) if serializer.is_valid(raise_exception=True): - extra_data = {"create_ticket": serializer.validated_data["create_ticket"]} + extra_data = serializer.validated_data logger.info( "Trigger Airflow DAG: %s for %s", ResolutionDags[serializer.validated_data["value"]], pk, ) + utils.add_decision(pk, request.user, serializer.validated_data["value"]) return airflow_utils.trigger_airflow_dag( ResolutionDags[serializer.validated_data["value"]].label, pk, extra_data diff --git a/backoffice/backoffice/workflows/constants.py b/backoffice/backoffice/workflows/constants.py index 273faeaca..6907dedaa 100644 --- a/backoffice/backoffice/workflows/constants.py +++ b/backoffice/backoffice/workflows/constants.py @@ -33,6 +33,10 @@ class WorkflowType(models.TextChoices): class ResolutionDags(models.TextChoices): accept = "accept", "author_create_approved_dag" reject = "reject", "author_create_rejected_dag" + accept_curate = "accept_curate", "author_create_approved_dag" + + +DECISION_CHOICES = ResolutionDags.choices class AuthorCreateDags(models.TextChoices): diff --git a/backoffice/backoffice/workflows/migrations/0009_decision.py b/backoffice/backoffice/workflows/migrations/0009_decision.py new file mode 100644 index 000000000..a33ca9245 --- /dev/null +++ b/backoffice/backoffice/workflows/migrations/0009_decision.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.6 on 2024-08-15 12:25 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("workflows", "0008_alter_workflow_status_alter_workflow_workflow_type"), + ] + + operations = [ + migrations.CreateModel( + name="Decision", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "action", + models.CharField( + choices=[ + ("accept", "author_create_approved_dag"), + ("reject", "author_create_rejected_dag"), + ("accept_curate", "author_create_approved_dag"), + ], + max_length=30, + ), + ), + ("_created_at", models.DateTimeField(auto_now_add=True)), + ("_updated_at", models.DateTimeField(auto_now=True)), + ( + "user", + models.ForeignKey( + db_column="email", + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + to_field="email", + ), + ), + ( + "workflow", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="workflows.workflow", + ), + ), + ], + ), + ] diff --git a/backoffice/backoffice/workflows/models.py b/backoffice/backoffice/workflows/models.py index cd63f4765..590c831db 100644 --- a/backoffice/backoffice/workflows/models.py +++ b/backoffice/backoffice/workflows/models.py @@ -2,7 +2,9 @@ from django.db import models +from backoffice.users.models import User from backoffice.workflows.constants import ( + DECISION_CHOICES, DEFAULT_STATUS_CHOICE, DEFAULT_TICKET_TYPE, DEFAULT_WORKFLOW_TYPE, @@ -43,3 +45,14 @@ class WorkflowTicket(models.Model): ticket_type = models.CharField( max_length=30, choices=TICKET_TYPES, default=DEFAULT_TICKET_TYPE ) + + +class Decision(models.Model): + user = models.ForeignKey( + User, to_field="email", db_column="email", on_delete=models.CASCADE + ) + workflow = models.ForeignKey(Workflow, on_delete=models.CASCADE) + action = models.CharField(max_length=30, choices=DECISION_CHOICES) + + _created_at = models.DateTimeField(auto_now_add=True) + _updated_at = models.DateTimeField(auto_now=True) diff --git a/backoffice/backoffice/workflows/tests/test_utils.py b/backoffice/backoffice/workflows/tests/test_utils.py new file mode 100644 index 000000000..dc297c3e1 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/test_utils.py @@ -0,0 +1,44 @@ +import uuid + +import pytest +from django.apps import apps +from django.contrib.auth import get_user_model +from django.test import TransactionTestCase +from rest_framework.exceptions import ValidationError + +from backoffice.workflows import constants +from backoffice.workflows.api import utils +from backoffice.workflows.constants import StatusChoices + +User = get_user_model() +Workflow = apps.get_model(app_label="workflows", model_name="Workflow") + + +class TestUtils(TransactionTestCase): + reset_sequences = True + fixtures = ["backoffice/fixtures/groups.json"] + + def setUp(self): + super().setUp() + self.workflow = Workflow.objects.create( + data={}, status=StatusChoices.APPROVAL, core=True, is_update=False + ) + self.user = User.objects.create_user( + email="testuser@test.com", password="12345" + ) + + def test_add_decision(self): + decision_data = utils.add_decision( + self.workflow.id, self.user, constants.ResolutionDags.accept + ) + + self.assertIsNotNone(decision_data) + + def test_add_decision_validation_errors(self): + with pytest.raises(ValidationError): + utils.add_decision(self.workflow.id, self.user, "wrong") + + with pytest.raises(ValidationError): + utils.add_decision( + uuid.UUID(int=0), self.user, constants.ResolutionDags.accept + ) diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 8775b0b61..8a9063e5f 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -11,6 +11,7 @@ from django.test import TransactionTestCase from django.urls import reverse from django_opensearch_dsl.registries import registry +from rest_framework import status from rest_framework.test import APIClient from backoffice.workflows import airflow_utils @@ -23,6 +24,7 @@ User = get_user_model() Workflow = apps.get_model(app_label="workflows", model_name="Workflow") +Decision = apps.get_model(app_label="workflows", model_name="Decision") class BaseTransactionTestCase(TransactionTestCase): @@ -322,7 +324,8 @@ def test_create_author(self): @pytest.mark.vcr() def test_accept_author(self): self.api_client.force_authenticate(user=self.curator) - data = {"create_ticket": True, "value": "accept"} + action = "accept" + data = {"create_ticket": True, "value": action} response = self.api_client.post( reverse("api:workflows-authors-resolve", kwargs={"pk": self.workflow.id}), @@ -331,6 +334,9 @@ def test_accept_author(self): ) self.assertEqual(response.status_code, 200) + self.assertEqual( + Decision.objects.filter(workflow=self.workflow.id)[0].action, action + ) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].approve, self.workflow.id @@ -339,7 +345,8 @@ def test_accept_author(self): @pytest.mark.vcr() def test_reject_author(self): self.api_client.force_authenticate(user=self.curator) - data = {"create_ticket": True, "value": "reject"} + action = "reject" + data = {"create_ticket": True, "value": action} response = self.api_client.post( reverse("api:workflows-authors-resolve", kwargs={"pk": self.workflow.id}), @@ -348,6 +355,9 @@ def test_reject_author(self): ) self.assertEqual(response.status_code, 200) + self.assertEqual( + Decision.objects.filter(workflow=self.workflow.id)[0].action, action + ) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].reject, self.workflow.id @@ -492,3 +502,26 @@ def test_ordering(self): if previous_date is not None: assert cur_date < previous_date previous_date = cur_date + + +class TestDecisionsViewSet(BaseTransactionTestCase): + endpoint = "/api/decisions" + reset_sequences = True + fixtures = ["backoffice/fixtures/groups.json"] + + def setUp(self): + super().setUp() + self.workflow = Workflow.objects.create( + data={}, status="running", core=True, is_update=False + ) + + def test_create_decision(self): + self.api_client.force_authenticate(user=self.curator) + data = { + "workflow_id": self.workflow.id, + "action": "accept", + } + + url = reverse("api:decisions-list") + response = self.api_client.post(url, format="json", data=data) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) diff --git a/backoffice/config/api_router.py b/backoffice/config/api_router.py index 179a686bb..fce8e177b 100644 --- a/backoffice/config/api_router.py +++ b/backoffice/config/api_router.py @@ -4,6 +4,7 @@ from backoffice.users.api.views import UserViewSet from backoffice.workflows.api.views import ( AuthorWorkflowViewSet, + DecisionViewSet, WorkflowTicketViewSet, WorkflowViewSet, ) @@ -20,5 +21,6 @@ ) router.register("workflows", WorkflowViewSet, basename="workflows") (router.register("workflow-ticket", WorkflowTicketViewSet, basename="workflow-ticket"),) +router.register("decisions", DecisionViewSet, basename="decisions") app_name = "api" urlpatterns = router.urls From f41292acd2a3c03aaeb95972044a54bdbfd3831a Mon Sep 17 00:00:00 2001 From: DonHaul Date: Fri, 9 Aug 2024 10:37:23 +0200 Subject: [PATCH 3/7] workflows: destroy workflows and delete/stop running tasks * ref: cern-sis/issues-inspire#532 --- .../backoffice/workflows/airflow_utils.py | 17 +- backoffice/backoffice/workflows/api/views.py | 4 + ...owUtils.test_delete_workflow_dag_runs.yaml | 209 ++++++++++ .../TestWorkflowViewSet.test_delete.yaml | 372 ++++++++++++++++++ .../workflows/tests/test_airflow_utils.py | 7 + .../backoffice/workflows/tests/test_views.py | 35 +- 6 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index 95f856269..5f88b9e7e 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -156,11 +156,24 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): :param params: parameters of new dag execution :returns: request response """ + delete_workflow_dag_runs(workflow_id, workflow_type) + + return trigger_airflow_dag( + WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params + ) + + +def delete_workflow_dag_runs(workflow_id, workflow_type): + """Deletes runs of a given workflow. + + :param workflow_id: workflow_id for dags that should be restarted + :param workflow_type: type of workflow the will be restarted + """ executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) for dag_id in executed_dags_for_workflow: delete_workflow_dag(dag_id, str(workflow_id)) - return trigger_airflow_dag( - WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params + return JsonResponse( + data={"message": f"Dag runs for worfklow {workflow_id} have been deleted"} ) diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index eb4652885..6c3fb6f09 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -53,6 +53,10 @@ def get_queryset(self): return self.queryset.filter(status__status=status) return self.queryset + def perform_destroy(self, instance): + airflow_utils.delete_workflow_dag_runs(instance.id, instance.workflow_type) + super().perform_destroy(instance) + class WorkflowTicketViewSet(viewsets.ViewSet): def retrieve(self, request, *args, **kwargs): diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml new file mode 100644 index 000000000..f997984a7 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml @@ -0,0 +1,209 @@ +interactions: +- request: + body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": + "00000000-0000-0000-0000-000000000001"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"Not Found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '293' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml new file mode 100644 index 000000000..73499d6fb --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml @@ -0,0 +1,372 @@ +interactions: +- request: + body: '{"dag_run_id": "00000000-0000-0000-0000-000000000002", "conf": {"workflow_id": + "00000000-0000-0000-0000-000000000002"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '300' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/test_airflow_utils.py b/backoffice/backoffice/workflows/tests/test_airflow_utils.py index 2d508c0bc..eb64b5418 100644 --- a/backoffice/backoffice/workflows/tests/test_airflow_utils.py +++ b/backoffice/backoffice/workflows/tests/test_airflow_utils.py @@ -64,3 +64,10 @@ def test_restart_workflow_dags(self): self.workflow_id, self.workflow_type ) self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_delete_workflow_dag_runs(self): + response = airflow_utils.delete_workflow_dag_runs( + self.workflow_id, self.workflow_type + ) + self.assertEqual(response.status_code, 200) diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 8a9063e5f..2cb09e1b4 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -19,7 +19,12 @@ WorkflowSerializer, WorkflowTicketSerializer, ) -from backoffice.workflows.constants import WORKFLOW_DAGS, StatusChoices, WorkflowType +from backoffice.workflows.constants import ( + WORKFLOW_DAGS, + AuthorCreateDags, + StatusChoices, + WorkflowType, +) from backoffice.workflows.models import WorkflowTicket User = get_user_model() @@ -57,7 +62,12 @@ class TestWorkflowViewSet(BaseTransactionTestCase): def setUp(self): super().setUp() self.workflow = Workflow.objects.create( - data={}, status=StatusChoices.APPROVAL, core=True, is_update=False + data={}, + status=StatusChoices.APPROVAL, + core=True, + is_update=False, + workflow_type=WorkflowType.AUTHOR_CREATE, + id=uuid.UUID(int=2), ) def test_list_curator(self): @@ -90,6 +100,27 @@ def test_tickets(self): assert "ticket_id" in workflow_data["tickets"][0] assert "ticket_type" in workflow_data["tickets"][0] + @pytest.mark.vcr() + def test_delete(self): + self.api_client.force_authenticate(user=self.curator) + airflow_utils.trigger_airflow_dag( + AuthorCreateDags.initialize, str(self.workflow.id) + ) + assert airflow_utils.find_executed_dags( + self.workflow.id, self.workflow.workflow_type + ) + + url = reverse("api:workflows-detail", kwargs={"pk": self.workflow.id}) + response = self.api_client.delete(url) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + assert ( + airflow_utils.find_executed_dags( + self.workflow.id, self.workflow.workflow_type + ) + == {} + ) + class TestWorkflowSearchViewSet(BaseTransactionTestCase): endpoint = "/api/workflows/search/" From ca17c2b63ecc822567d7595dd685c2a104a003d1 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Fri, 23 Aug 2024 14:47:16 +0200 Subject: [PATCH 4/7] #draft backoffice: added decision connecion in serializer * ref: cern-sis/issues-inspire/issues/518 --- .../backoffice/workflows/api/serializers.py | 13 ++++---- .../0010_alter_decision_workflow_and_more.py | 31 +++++++++++++++++++ backoffice/backoffice/workflows/models.py | 9 ++++-- .../backoffice/workflows/tests/test_views.py | 10 ++++++ 4 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py diff --git a/backoffice/backoffice/workflows/api/serializers.py b/backoffice/backoffice/workflows/api/serializers.py index 4883d0b20..bf7bf9f57 100644 --- a/backoffice/backoffice/workflows/api/serializers.py +++ b/backoffice/backoffice/workflows/api/serializers.py @@ -24,19 +24,20 @@ def get_ticket_url(self, obj): ) -class WorkflowSerializer(serializers.ModelSerializer): - tickets = WorkflowTicketSerializer(many=True, read_only=True) +class DecisionSerializer(serializers.ModelSerializer): + workflow = serializers.PrimaryKeyRelatedField(queryset=Workflow.objects.all()) class Meta: - model = Workflow + model = Decision fields = "__all__" -class DecisionSerializer(serializers.ModelSerializer): - workflow = serializers.PrimaryKeyRelatedField(queryset=Workflow.objects.all()) +class WorkflowSerializer(serializers.ModelSerializer): + tickets = WorkflowTicketSerializer(many=True, read_only=True) + decisions = DecisionSerializer(many=True, read_only=True) class Meta: - model = Decision + model = Workflow fields = "__all__" diff --git a/backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py b/backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py new file mode 100644 index 000000000..a9cefea0e --- /dev/null +++ b/backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.6 on 2024-08-23 13:02 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("workflows", "0009_decision"), + ] + + operations = [ + migrations.AlterField( + model_name="decision", + name="workflow", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="decisions", + to="workflows.workflow", + ), + ), + migrations.AlterField( + model_name="workflowticket", + name="workflow_id", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="tickets", + to="workflows.workflow", + ), + ), + ] diff --git a/backoffice/backoffice/workflows/models.py b/backoffice/backoffice/workflows/models.py index 590c831db..d4fb37049 100644 --- a/backoffice/backoffice/workflows/models.py +++ b/backoffice/backoffice/workflows/models.py @@ -49,9 +49,14 @@ class WorkflowTicket(models.Model): class Decision(models.Model): user = models.ForeignKey( - User, to_field="email", db_column="email", on_delete=models.CASCADE + User, + to_field="email", + db_column="email", + on_delete=models.CASCADE, + ) + workflow = models.ForeignKey( + Workflow, related_name="decisions", on_delete=models.CASCADE ) - workflow = models.ForeignKey(Workflow, on_delete=models.CASCADE) action = models.CharField(max_length=30, choices=DECISION_CHOICES) _created_at = models.DateTimeField(auto_now_add=True) diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index a0ac4aab9..d9395e95d 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -22,6 +22,7 @@ from backoffice.workflows.constants import ( WORKFLOW_DAGS, AuthorCreateDags, + ResolutionDags, StatusChoices, WorkflowType, ) @@ -100,6 +101,15 @@ def test_tickets(self): assert "ticket_id" in workflow_data["tickets"][0] assert "ticket_type" in workflow_data["tickets"][0] + def test_decisions(self): + Decision.objects.create( + workflow=self.workflow, user=self.user, action=ResolutionDags.accept + ) + workflow_data = WorkflowSerializer(self.workflow).data + assert "decisions" in workflow_data + assert "action" in workflow_data["decisions"][0] + assert "user" in workflow_data["decisions"][0] + @pytest.mark.vcr() def test_delete(self): self.api_client.force_authenticate(user=self.curator) From 9fb297abbd77ebf53ca8eca58f1d3ae70e1be55e Mon Sep 17 00:00:00 2001 From: DonHaul Date: Fri, 23 Aug 2024 16:58:52 +0200 Subject: [PATCH 5/7] backoffice: ordering by best match added * ref: cern-sis/issues-inspire/issues/531 --- backoffice/backoffice/workflows/api/views.py | 4 +-- .../backoffice/workflows/tests/test_views.py | 30 ++++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 2a67bfdd6..17172077f 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -272,9 +272,9 @@ def __init__(self, *args, **kwargs): "is_update": "is_update", } - ordering_fields = {"_updated_at": "_updated_at"} + ordering_fields = {"_updated_at": "_updated_at", "_score": "_score"} - ordering = ("-_updated_at",) + ordering = ("-_updated_at", "-_score") faceted_search_fields = { "status": { diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index d9395e95d..319d6c94e 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -447,12 +447,15 @@ class TestWorkflowSearchFilterViewSet(BaseTransactionTestCase): reset_sequences = True fixtures = ["backoffice/fixtures/groups.json"] - def setUp(self): - super().setUp() - @classmethod def setUpClass(cls): super().setUpClass() + + index = registry.get_indices().pop() + with contextlib.suppress(opensearchpy.exceptions.NotFoundError): + index.delete() + index.create() + Workflow.objects.update_or_create( data={ "ids": [ @@ -528,7 +531,7 @@ def test_filter_workflow_type(self): for item in response.json()["results"]: assert item["workflow_type"] == WorkflowType.AUTHOR_CREATE - def test_ordering(self): + def test_ordering_updated_at(self): self.api_client.force_authenticate(user=self.admin) base_url = reverse("search:workflow-list") @@ -545,6 +548,25 @@ def test_ordering(self): assert cur_date < previous_date previous_date = cur_date + def test_ordering_score(self): + self.api_client.force_authenticate(user=self.admin) + + search_str = "search=Frank Castle^10 OR John^6" + + url = reverse("search:workflow-list") + f"?ordering=_score&{search_str}" + response = self.api_client.get(url) + self.assertEqual( + response.json()["results"][0]["data"]["name"]["preferred_name"], + "John Smith", + ) + + url = reverse("search:workflow-list") + f"?ordering=-_score&{search_str}" + response = self.api_client.get(url) + self.assertEqual( + response.json()["results"][0]["data"]["name"]["preferred_name"], + "Frank Castle", + ) + class TestDecisionsViewSet(BaseTransactionTestCase): endpoint = "/api/decisions" From 764dffda05dc9fc94c5d42d422931058ca97c97e Mon Sep 17 00:00:00 2001 From: DonHaul Date: Mon, 26 Aug 2024 16:08:58 +0200 Subject: [PATCH 6/7] opensearch: auto index upon container start * ref: cern-sis/issues-inspire/issues/530 --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3f33162d1..a4a2e6b1d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -104,7 +104,7 @@ services: - ./backoffice/.envs/local/.postgres ports: - "8000:8000" - command: sh -c 'poetry run python manage.py migrate && poetry run python manage.py opensearch index create --force && poetry run python manage.py runserver 0.0.0.0:8000' + command: sh -c 'poetry run python manage.py migrate && poetry run python manage.py opensearch index rebuild --force && poetry run python manage.py opensearch document index --force && poetry run python manage.py runserver 0.0.0.0:8000' postgres-airflow: image: postgres:13 From 97fd4cf50d4e57534ecfad4710a36eb94a08ecaf Mon Sep 17 00:00:00 2001 From: DonHaul Date: Mon, 26 Aug 2024 16:40:40 +0200 Subject: [PATCH 7/7] backoffice: workflow views return object * ref: cern-sis/issues-inspire/issues/553 --- backoffice/backoffice/workflows/api/views.py | 10 ++++++++-- backoffice/backoffice/workflows/tests/test_views.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 2a67bfdd6..fafff4c8c 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -122,11 +122,12 @@ def create(self, request): WORKFLOW_DAGS[workflow.workflow_type].initialize, workflow.id, ) - return airflow_utils.trigger_airflow_dag( + airflow_utils.trigger_airflow_dag( WORKFLOW_DAGS[workflow.workflow_type].initialize, str(workflow.id), workflow.data, ) + return Response(serializer.data, status=status.HTTP_201_CREATED) @extend_schema( summary="Partially Updates Author", @@ -169,9 +170,14 @@ def resolve(self, request, pk=None): ) utils.add_decision(pk, request.user, serializer.validated_data["value"]) - return airflow_utils.trigger_airflow_dag( + airflow_utils.trigger_airflow_dag( ResolutionDags[serializer.validated_data["value"]].label, pk, extra_data ) + workflow_serializer = self.serializer_class( + get_object_or_404(Workflow, pk=pk) + ) + + return Response(workflow_serializer.data) @extend_schema( summary="Restart an Author Workflow", diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index d9395e95d..1f3919b97 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -215,6 +215,8 @@ def test_patch_admin(self): "test": "test", }, ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) def test_patch_anonymous(self): self.api_client.force_authenticate(user=self.user) @@ -361,7 +363,8 @@ def test_create_author(self): url = reverse("api:workflows-authors-list") response = self.api_client.post(url, format="json", data=data) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json(), data) @pytest.mark.vcr() def test_accept_author(self): @@ -379,7 +382,8 @@ def test_accept_author(self): self.assertEqual( Decision.objects.filter(workflow=self.workflow.id)[0].action, action ) - + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].approve, self.workflow.id ) @@ -400,6 +404,8 @@ def test_reject_author(self): self.assertEqual( Decision.objects.filter(workflow=self.workflow.id)[0].action, action ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].reject, self.workflow.id