diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index 95f85626..5f88b9e7 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -156,11 +156,24 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): :param params: parameters of new dag execution :returns: request response """ + delete_workflow_dag_runs(workflow_id, workflow_type) + + return trigger_airflow_dag( + WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params + ) + + +def delete_workflow_dag_runs(workflow_id, workflow_type): + """Deletes runs of a given workflow. + + :param workflow_id: workflow_id for dags that should be restarted + :param workflow_type: type of workflow the will be restarted + """ executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) for dag_id in executed_dags_for_workflow: delete_workflow_dag(dag_id, str(workflow_id)) - return trigger_airflow_dag( - WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params + return JsonResponse( + data={"message": f"Dag runs for worfklow {workflow_id} have been deleted"} ) diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 5ddd5881..140225f9 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -52,6 +52,10 @@ def get_queryset(self): return self.queryset.filter(status__status=status) return self.queryset + def perform_destroy(self, instance): + airflow_utils.delete_workflow_dag_runs(instance.id, instance.workflow_type) + super().perform_destroy(instance) + class WorkflowTicketViewSet(viewsets.ViewSet): def retrieve(self, request, *args, **kwargs): diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml new file mode 100644 index 00000000..f997984a --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml @@ -0,0 +1,209 @@ +interactions: +- request: + body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": + "00000000-0000-0000-0000-000000000001"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"Not Found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '293' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml new file mode 100644 index 00000000..73499d6f --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml @@ -0,0 +1,372 @@ +interactions: +- request: + body: '{"dag_run_id": "00000000-0000-0000-0000-000000000002", "conf": {"workflow_id": + "00000000-0000-0000-0000-000000000002"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '300' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/test_airflow_utils.py b/backoffice/backoffice/workflows/tests/test_airflow_utils.py index 2d508c0b..eb64b541 100644 --- a/backoffice/backoffice/workflows/tests/test_airflow_utils.py +++ b/backoffice/backoffice/workflows/tests/test_airflow_utils.py @@ -64,3 +64,10 @@ def test_restart_workflow_dags(self): self.workflow_id, self.workflow_type ) self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_delete_workflow_dag_runs(self): + response = airflow_utils.delete_workflow_dag_runs( + self.workflow_id, self.workflow_type + ) + self.assertEqual(response.status_code, 200) diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 9127ada4..0b131a75 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -11,6 +11,7 @@ from django.test import TransactionTestCase from django.urls import reverse from django_opensearch_dsl.registries import registry +from rest_framework import status from rest_framework.test import APIClient from backoffice.workflows import airflow_utils @@ -18,7 +19,12 @@ WorkflowSerializer, WorkflowTicketSerializer, ) -from backoffice.workflows.constants import WORKFLOW_DAGS, StatusChoices, WorkflowType +from backoffice.workflows.constants import ( + WORKFLOW_DAGS, + AuthorCreateDags, + StatusChoices, + WorkflowType, +) from backoffice.workflows.models import WorkflowTicket User = get_user_model() @@ -55,7 +61,12 @@ class TestWorkflowViewSet(BaseTransactionTestCase): def setUp(self): super().setUp() self.workflow = Workflow.objects.create( - data={}, status=StatusChoices.APPROVAL, core=True, is_update=False + data={}, + status=StatusChoices.APPROVAL, + core=True, + is_update=False, + workflow_type=WorkflowType.AUTHOR_CREATE, + id=uuid.UUID(int=2), ) def test_list_curator(self): @@ -88,6 +99,27 @@ def test_tickets(self): assert "ticket_id" in workflow_data["tickets"][0] assert "ticket_type" in workflow_data["tickets"][0] + @pytest.mark.vcr() + def test_delete(self): + self.api_client.force_authenticate(user=self.curator) + airflow_utils.trigger_airflow_dag( + AuthorCreateDags.initialize, str(self.workflow.id) + ) + assert airflow_utils.find_executed_dags( + self.workflow.id, self.workflow.workflow_type + ) + + url = reverse("api:workflows-detail", kwargs={"pk": self.workflow.id}) + response = self.api_client.delete(url) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + assert ( + airflow_utils.find_executed_dags( + self.workflow.id, self.workflow.workflow_type + ) + == {} + ) + class TestWorkflowSearchViewSet(BaseTransactionTestCase): endpoint = "/api/workflows/search/"