From 1d03468d1ad23068bf19cdc7097299b443bc4887 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Mon, 29 Jul 2024 15:22:07 +0200 Subject: [PATCH] restart actions: added test for airflow utils --- .../backoffice/workflows/airflow_utils.py | 13 +- ...horWorkflowViewSet.test_accept_author.yaml | 145 ------------ ...horWorkflowViewSet.test_create_author.yaml | 85 ------- ...horWorkflowViewSet.test_reject_author.yaml | 145 ------------ ...orWorkflowViewSet.test_restart_a_task.yaml | 221 ------------------ ...kflowViewSet.test_restart_full_dagrun.yaml | 220 ----------------- ...kflowViewSet.test_restart_with_params.yaml | 220 ----------------- .../workflows/tests/test_airflow_utils.py | 62 +++++ 8 files changed, 68 insertions(+), 1043 deletions(-) delete mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_accept_author.yaml delete mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_create_author.yaml delete mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_reject_author.yaml delete mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_a_task.yaml delete mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml delete mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_with_params.yaml create mode 100644 backoffice/backoffice/workflows/tests/test_airflow_utils.py diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index df34899d..d97ef4b4 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -2,7 +2,7 @@ from os import environ import requests -from django.http import JsonResponse +from django.http import HttpResponse, JsonResponse from requests.exceptions import RequestException from rest_framework import status @@ -57,7 +57,6 @@ def restart_failed_tasks(workflow): """ dag_id = find_failed_dag(workflow) - # assumes current task is one of the failed tasks data = { "dry_run": False, @@ -94,7 +93,7 @@ def find_executed_dags(workflow): headers=AIRFLOW_HEADERS, ) if response.status_code == status.HTTP_200_OK: - executed_dags_for_workflow[dag_id] = response.content + executed_dags_for_workflow[dag_id] = response.json() return executed_dags_for_workflow @@ -109,7 +108,7 @@ def find_failed_dag(workflow): executed_dags_for_workflow = find_executed_dags(workflow) for dag, dag_data in executed_dags_for_workflow.items(): - if dag_data["status"] == "failed": + if dag_data["state"] == "failed": return dag @@ -125,12 +124,12 @@ def delete_workflow_dag(dag_id, workflow): headers=AIRFLOW_HEADERS, ) response.raise_for_status() + return HttpResponse() except RequestException: - data = {"error": response.json()} - return JsonResponse(data, status=status.HTTP_424_FAILED_DEPENDENCY) + return HttpResponse(status=status.HTTP_424_FAILED_DEPENDENCY) -def restart_workflow_dags(workflow, params): +def restart_workflow_dags(workflow, params=None): """Restarts dags of a given workflow. :param workflow: workflow whoose dags should be restarted diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_accept_author.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_accept_author.yaml deleted file mode 100644 index 77e4d8dc..00000000 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_accept_author.yaml +++ /dev/null @@ -1,145 +0,0 @@ -interactions: -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' - and DAGRun ID: '00000000-0000-0000-0000-000000000000' already exists\",\n - \ \"status\": 409,\n \"title\": \"Conflict\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/AlreadyExists\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '302' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:46:58 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 409 - message: CONFLICT -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:58 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000", "create_ticket": true}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '142' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns - response: - body: - string: "{\n \"conf\": {\n \"create_ticket\": true,\n \"workflow_id\": - \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": \"author_create_approved_dag\",\n - \ \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": - \"2024-07-29T07:46:58.558021+00:00\",\n \"data_interval_start\": \"2024-07-29T07:46:58.558021+00:00\",\n - \ \"end_date\": null,\n \"execution_date\": \"2024-07-29T07:46:58.558021+00:00\",\n - \ \"external_trigger\": true,\n \"last_scheduling_decision\": null,\n \"logical_date\": - \"2024-07-29T07:46:58.558021+00:00\",\n \"note\": null,\n \"run_type\": - \"manual\",\n \"start_date\": null,\n \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '600' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:58 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:58 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_create_author.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_create_author.yaml deleted file mode 100644 index 308440d8..00000000 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_create_author.yaml +++ /dev/null @@ -1,85 +0,0 @@ -interactions: -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' - and DAGRun ID: '00000000-0000-0000-0000-000000000000' already exists\",\n - \ \"status\": 409,\n \"title\": \"Conflict\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/AlreadyExists\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '302' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:46:58 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 409 - message: CONFLICT -- request: - body: '{"dag_run_id": "eac8ef74-04a2-44ad-8ccd-eaa05fada3cb", "conf": {"workflow_id": - "eac8ef74-04a2-44ad-8ccd-eaa05fada3cb", "native_name": "NATIVE_NAME", "alternate_name": - "NAME", "display_name": "FIRST_NAME", "family_name": "LAST_NAME", "given_name": - "GIVEN_NAME"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '261' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"conf\": {\n \"alternate_name\": \"NAME\",\n \"display_name\": - \"FIRST_NAME\",\n \"family_name\": \"LAST_NAME\",\n \"given_name\": - \"GIVEN_NAME\",\n \"native_name\": \"NATIVE_NAME\",\n \"workflow_id\": - \"eac8ef74-04a2-44ad-8ccd-eaa05fada3cb\"\n },\n \"dag_id\": \"author_create_initialization_dag\",\n - \ \"dag_run_id\": \"eac8ef74-04a2-44ad-8ccd-eaa05fada3cb\",\n \"data_interval_end\": - \"2024-07-29T07:46:58.907705+00:00\",\n \"data_interval_start\": \"2024-07-29T07:46:58.907705+00:00\",\n - \ \"end_date\": null,\n \"execution_date\": \"2024-07-29T07:46:58.907705+00:00\",\n - \ \"external_trigger\": true,\n \"last_scheduling_decision\": null,\n \"logical_date\": - \"2024-07-29T07:46:58.907705+00:00\",\n \"note\": null,\n \"run_type\": - \"manual\",\n \"start_date\": null,\n \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '741' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:58 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_reject_author.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_reject_author.yaml deleted file mode 100644 index 1135954f..00000000 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_reject_author.yaml +++ /dev/null @@ -1,145 +0,0 @@ -interactions: -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' - and DAGRun ID: '00000000-0000-0000-0000-000000000000' already exists\",\n - \ \"status\": 409,\n \"title\": \"Conflict\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/AlreadyExists\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '302' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 409 - message: CONFLICT -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000", "create_ticket": true}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '142' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns - response: - body: - string: "{\n \"conf\": {\n \"create_ticket\": true,\n \"workflow_id\": - \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": \"author_create_rejected_dag\",\n - \ \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": - \"2024-07-29T07:46:59.199305+00:00\",\n \"data_interval_start\": \"2024-07-29T07:46:59.199305+00:00\",\n - \ \"end_date\": null,\n \"execution_date\": \"2024-07-29T07:46:59.199305+00:00\",\n - \ \"external_trigger\": true,\n \"last_scheduling_decision\": null,\n \"logical_date\": - \"2024-07-29T07:46:59.199305+00:00\",\n \"note\": null,\n \"run_type\": - \"manual\",\n \"start_date\": null,\n \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '600' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_a_task.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_a_task.yaml deleted file mode 100644 index 86af1282..00000000 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_a_task.yaml +++ /dev/null @@ -1,221 +0,0 @@ -interactions: -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' - and DAGRun ID: '00000000-0000-0000-0000-000000000000' already exists\",\n - \ \"status\": 409,\n \"title\": \"Conflict\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/AlreadyExists\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '302' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 409 - message: CONFLICT -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-29T07:25:51.289983+00:00\",\n - \ \"data_interval_start\": \"2024-07-29T07:25:51.289983+00:00\",\n \"end_date\": - \"2024-07-29T07:46:25.323270+00:00\",\n \"execution_date\": \"2024-07-29T07:25:51.289983+00:00\",\n - \ \"external_trigger\": true,\n \"last_scheduling_decision\": \"2024-07-29T07:46:25.320269+00:00\",\n - \ \"logical_date\": \"2024-07-29T07:25:51.289983+00:00\",\n \"note\": null,\n - \ \"run_type\": \"manual\",\n \"start_date\": \"2024-07-29T07:46:07.283489+00:00\",\n - \ \"state\": \"failed\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '669' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' - and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '294' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 404 - message: NOT FOUND -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' - and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '294' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 404 - message: NOT FOUND -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-29T07:46:59.795765+00:00\",\n - \ \"data_interval_start\": \"2024-07-29T07:46:59.795765+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-29T07:46:59.795765+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T07:46:59.795765+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '579' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:46:59 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml deleted file mode 100644 index c1c1880a..00000000 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml +++ /dev/null @@ -1,220 +0,0 @@ -interactions: -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' - and DAGRun ID: '00000000-0000-0000-0000-000000000000' already exists\",\n - \ \"status\": 409,\n \"title\": \"Conflict\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/AlreadyExists\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '302' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 409 - message: CONFLICT -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-29T07:46:59.795765+00:00\",\n - \ \"data_interval_start\": \"2024-07-29T07:46:59.795765+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-29T07:46:59.795765+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T07:46:59.795765+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '579' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' - and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '294' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 404 - message: NOT FOUND -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' - and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '294' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 404 - message: NOT FOUND -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-29T07:47:00.334741+00:00\",\n - \ \"data_interval_start\": \"2024-07-29T07:47:00.334741+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-29T07:47:00.334741+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T07:47:00.334741+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '579' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_with_params.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_with_params.yaml deleted file mode 100644 index dbdb235a..00000000 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_with_params.yaml +++ /dev/null @@ -1,220 +0,0 @@ -interactions: -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' - and DAGRun ID: '00000000-0000-0000-0000-000000000000' already exists\",\n - \ \"status\": 409,\n \"title\": \"Conflict\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/AlreadyExists\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '302' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 409 - message: CONFLICT -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-29T07:47:00.334741+00:00\",\n - \ \"data_interval_start\": \"2024-07-29T07:47:00.334741+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-29T07:47:00.334741+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T07:47:00.334741+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '579' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' - and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '294' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 404 - message: NOT FOUND -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Type: - - application/json - method: GET - uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' - and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '294' - Content-Type: - - application/problem+json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 404 - message: NOT FOUND -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '0' - Content-Type: - - application/json - method: DELETE - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 - response: - body: - string: '' - headers: - Connection: - - close - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 204 - message: NO CONTENT -- request: - body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '119' - Content-Type: - - application/json - method: POST - uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns - response: - body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-29T07:47:00.815376+00:00\",\n - \ \"data_interval_start\": \"2024-07-29T07:47:00.815376+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-29T07:47:00.815376+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T07:47:00.815376+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" - headers: - Connection: - - close - Content-Length: - - '579' - Content-Type: - - application/json - Date: - - Mon, 29 Jul 2024 07:47:00 GMT - Server: - - gunicorn - X-Robots-Tag: - - noindex, nofollow - status: - code: 200 - message: OK -version: 1 diff --git a/backoffice/backoffice/workflows/tests/test_airflow_utils.py b/backoffice/backoffice/workflows/tests/test_airflow_utils.py new file mode 100644 index 00000000..c568ccbb --- /dev/null +++ b/backoffice/backoffice/workflows/tests/test_airflow_utils.py @@ -0,0 +1,62 @@ +import time +import uuid + +import pytest +from django.apps import apps +from django.test import TransactionTestCase + +from backoffice.workflows import airflow_utils +from backoffice.workflows.constants import WORKFLOW_DAGS, StatusChoices, WorkflowType + +Workflow = apps.get_model(app_label="workflows", model_name="Workflow") + + +class TestAirflowUtils(TransactionTestCase): + def setUp(self): + self.dag_id = WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE][0] + self.workflow = Workflow.objects.create( + data={}, + status=StatusChoices.APPROVAL, + core=True, + is_update=False, + workflow_type=WorkflowType.AUTHOR_CREATE, + id=uuid.UUID(int=1), + ) + self.response = airflow_utils.trigger_airflow_dag( + self.dag_id, str(self.workflow.id) + ) + + def tearDown(self) -> None: + airflow_utils.delete_workflow_dag(self.dag_id, self.workflow) + + @pytest.mark.vcr() + def test_trigger_airflow_dag(self): + self.assertEqual(self.response.status_code, 200) + + @pytest.mark.vcr() + def test_restart_failed_tasks(self): + time.sleep(20) # wait for dag to fail + response = airflow_utils.restart_failed_tasks(self.workflow) + self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_find_executed_dags(self): + executed_dags_for_workflow = airflow_utils.find_executed_dags(self.workflow) + + self.assertIn(self.dag_id, executed_dags_for_workflow) + + @pytest.mark.vcr() + def test_find_failed_dag(self): + time.sleep(20) # wait for dag to fail + failed_dag = airflow_utils.find_failed_dag(self.workflow) + self.assertEqual(self.dag_id, failed_dag) + + @pytest.mark.vcr() + def test_delete_workflow_dag(self): + response = airflow_utils.delete_workflow_dag(self.dag_id, self.workflow) + self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_restart_workflow_dags(self): + response = airflow_utils.restart_workflow_dags(self.workflow) + self.assertEqual(response.status_code, 200)