user actions: make it independent from django

inspirehep · Jul 30, 2024 · db49239 · db49239
1 parent 7ee5c32
commit db49239
Show file tree

Hide file tree

Showing 14 changed files with 171 additions and 162 deletions.
diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py
@@ -49,25 +49,34 @@ def trigger_airflow_dag(dag_id, workflow_id, extra_data=None):
         return JsonResponse(data, status=status.HTTP_502_BAD_GATEWAY)
 
 
-def restart_failed_tasks(workflow):
+def restart_failed_tasks(workflow_id, workflow_type):
     """Restarts failed tasks of an airflow dag.
 
     :param workflow: workflow whooses tasks should be restarted
     :returns: request response
     """
 
-    dag_id = find_failed_dag(workflow)
+    dag_id = find_failed_dag(workflow_id, workflow_type)
     #  assumes current task is one of the failed tasks
     data = {
         "dry_run": False,
-        "dag_run_id": str(workflow.id),
+        "dag_run_id": str(workflow_id),
         "reset_dag_runs": False,
         "only_failed": True,
     }
 
+    url = f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/clearTaskInstances"
+
     try:
+        logger.info(
+            "Clearing Failed Tasks of DAG %s with data: %s and %s %s",
+            dag_id,
+            data,
+            AIRFLOW_HEADERS,
+            url,
+        )
         response = requests.post(
-            f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/clearTaskInstances",
+            url,
             json=data,
             headers=AIRFLOW_HEADERS,
         )
@@ -78,7 +87,7 @@ def restart_failed_tasks(workflow):
         return JsonResponse(data, status=status.HTTP_424_FAILED_DEPENDENCY)
 
 
-def find_executed_dags(workflow):
+def find_executed_dags(workflow_id, workflow_type):
     """For a given workflow find dags associated to it
 
     :param workflow: workflow to look dags for
@@ -87,9 +96,9 @@ def find_executed_dags(workflow):
 
     executed_dags_for_workflow = {}
     # find dags that were executed
-    for dag_id in WORKFLOW_DAGS[workflow.workflow_type]:
+    for dag_id in WORKFLOW_DAGS[workflow_type]:
         response = requests.get(
-            f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/dagRuns/{workflow.id}",
+            f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/dagRuns/{workflow_id}",
             headers=AIRFLOW_HEADERS,
         )
         if response.status_code == status.HTTP_200_OK:
@@ -98,29 +107,29 @@ def find_executed_dags(workflow):
     return executed_dags_for_workflow
 
 
-def find_failed_dag(workflow):
+def find_failed_dag(workflow_id, workflow_type):
     """For a given workflow find failed dags
 
     :param workflow: workflow to get failed dags
     :returns: failed dag id or none
     """
 
-    executed_dags_for_workflow = find_executed_dags(workflow)
+    executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type)
 
     for dag, dag_data in executed_dags_for_workflow.items():
         if dag_data["state"] == "failed":
             return dag
 
 
-def delete_workflow_dag(dag_id, workflow):
+def delete_workflow_dag(dag_id, workflow_id):
     """Delete dag run
     :param dag_id: dag to be removed
     :param workflow: workflow with the dag execution to be deleted
     :returns: request response
     """
     try:
         response = requests.delete(
-            f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/dagRuns/{str(workflow.id)}",
+            f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/dagRuns/{str(workflow_id)}",
             headers=AIRFLOW_HEADERS,
         )
         response.raise_for_status()
@@ -129,21 +138,20 @@ def delete_workflow_dag(dag_id, workflow):
         return HttpResponse(status=status.HTTP_424_FAILED_DEPENDENCY)
 
 
-def restart_workflow_dags(workflow, params=None):
+def restart_workflow_dags(workflow_id, workflow_type, params=None):
     """Restarts dags of a given workflow.
 
     :param workflow: workflow whoose dags should be restarted
     :param params: parameters of new dag execution
     :returns: request response
     """
-    executed_dags_for_workflow = find_executed_dags(workflow)
+    executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type)
 
     for dag_id in executed_dags_for_workflow:
-        #  delete all executions of workflow
-        delete_workflow_dag(dag_id, workflow)
+        delete_workflow_dag(dag_id, workflow_id)
 
         return trigger_airflow_dag(
-            WORKFLOW_DAGS[workflow.workflow_type][0], str(workflow.id), params
+            WORKFLOW_DAGS[workflow_type][0], str(workflow_id), params
         )
 
     return JsonResponse(

diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py
@@ -134,7 +134,9 @@ def restart(self, request, pk=None):
         if request.data.get("restart_current_task"):
             return airflow_utils.restart_failed_tasks(workflow)
 
-        return airflow_utils.restart_workflow_dags(workflow, request.data.get("params"))
+        return airflow_utils.restart_workflow_dags(
+            workflow.id, workflow.workflow_type, request.data.get("params")
+        )
 
 
 class WorkflowDocumentView(BaseDocumentViewSet):

diff --git a/backoffice/backoffice/workflows/constants.py b/backoffice/backoffice/workflows/constants.py
@@ -35,7 +35,6 @@ class ResolutionDags(models.TextChoices):
     reject = "reject", "author_create_rejected_dag"
 
 
-# author dags for each workflow type
 WORKFLOW_DAGS = {
     WorkflowType.HEP_CREATE: "",
     WorkflowType.HEP_UPDATE: "",

diff --git a/...ffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml b/...ffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml
@@ -19,10 +19,10 @@ interactions:
     body:
       string: "{\n  \"conf\": {\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
         \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
-        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-29T14:36:10.637820+00:00\",\n
-        \ \"data_interval_start\": \"2024-07-29T14:36:10.637820+00:00\",\n  \"end_date\":
-        null,\n  \"execution_date\": \"2024-07-29T14:36:10.637820+00:00\",\n  \"external_trigger\":
-        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-29T14:36:10.637820+00:00\",\n
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-30T08:29:54.362977+00:00\",\n
+        \ \"data_interval_start\": \"2024-07-30T08:29:54.362977+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-07-30T08:29:54.362977+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-30T08:29:54.362977+00:00\",\n
         \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
         \ \"state\": \"queued\"\n}\n"
     headers:
@@ -33,7 +33,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:10 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -65,7 +65,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:10 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -101,7 +101,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Mon, 29 Jul 2024 14:36:10 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:

diff --git a/...office/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml b/...office/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml
@@ -19,10 +19,10 @@ interactions:
     body:
       string: "{\n  \"conf\": {\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
         \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
-        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-29T14:36:10.935128+00:00\",\n
-        \ \"data_interval_start\": \"2024-07-29T14:36:10.935128+00:00\",\n  \"end_date\":
-        null,\n  \"execution_date\": \"2024-07-29T14:36:10.935128+00:00\",\n  \"external_trigger\":
-        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-29T14:36:10.935128+00:00\",\n
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-30T08:29:54.646527+00:00\",\n
+        \ \"data_interval_start\": \"2024-07-30T08:29:54.646527+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-07-30T08:29:54.646527+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-30T08:29:54.646527+00:00\",\n
         \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
         \ \"state\": \"queued\"\n}\n"
     headers:
@@ -33,7 +33,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:10 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -58,22 +58,21 @@ interactions:
     body:
       string: "{\n  \"conf\": {\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
         \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
-        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-29T14:36:10.935128+00:00\",\n
-        \ \"data_interval_start\": \"2024-07-29T14:36:10.935128+00:00\",\n  \"end_date\":
-        null,\n  \"execution_date\": \"2024-07-29T14:36:10.935128+00:00\",\n  \"external_trigger\":
-        true,\n  \"last_scheduling_decision\": \"2024-07-29T14:36:10.979214+00:00\",\n
-        \ \"logical_date\": \"2024-07-29T14:36:10.935128+00:00\",\n  \"note\": null,\n
-        \ \"run_type\": \"manual\",\n  \"start_date\": \"2024-07-29T14:36:10.977019+00:00\",\n
-        \ \"state\": \"running\"\n}\n"
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-30T08:29:54.646527+00:00\",\n
+        \ \"data_interval_start\": \"2024-07-30T08:29:54.646527+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-07-30T08:29:54.646527+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-30T08:29:54.646527+00:00\",\n
+        \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
+        \ \"state\": \"queued\"\n}\n"
     headers:
       Connection:
       - close
       Content-Length:
-      - '640'
+      - '579'
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:10 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -107,7 +106,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Mon, 29 Jul 2024 14:36:11 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -141,7 +140,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Mon, 29 Jul 2024 14:36:11 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -173,7 +172,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:11 GMT
+      - Tue, 30 Jul 2024 08:29:54 GMT
       Server:
       - gunicorn
       X-Robots-Tag:

diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_failed_dag.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_failed_dag.yaml
@@ -19,10 +19,10 @@ interactions:
     body:
       string: "{\n  \"conf\": {\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
         \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
-        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-29T14:36:11.363586+00:00\",\n
-        \ \"data_interval_start\": \"2024-07-29T14:36:11.363586+00:00\",\n  \"end_date\":
-        null,\n  \"execution_date\": \"2024-07-29T14:36:11.363586+00:00\",\n  \"external_trigger\":
-        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-29T14:36:11.363586+00:00\",\n
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-30T08:29:55.039650+00:00\",\n
+        \ \"data_interval_start\": \"2024-07-30T08:29:55.039650+00:00\",\n  \"end_date\":
+        null,\n  \"execution_date\": \"2024-07-30T08:29:55.039650+00:00\",\n  \"external_trigger\":
+        true,\n  \"last_scheduling_decision\": null,\n  \"logical_date\": \"2024-07-30T08:29:55.039650+00:00\",\n
         \ \"note\": null,\n  \"run_type\": \"manual\",\n  \"start_date\": null,\n
         \ \"state\": \"queued\"\n}\n"
     headers:
@@ -33,7 +33,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:11 GMT
+      - Tue, 30 Jul 2024 08:29:55 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -58,12 +58,12 @@ interactions:
     body:
       string: "{\n  \"conf\": {\n    \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n
         \ },\n  \"dag_id\": \"author_create_initialization_dag\",\n  \"dag_run_id\":
-        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-29T14:36:11.363586+00:00\",\n
-        \ \"data_interval_start\": \"2024-07-29T14:36:11.363586+00:00\",\n  \"end_date\":
-        \"2024-07-29T14:36:30.104727+00:00\",\n  \"execution_date\": \"2024-07-29T14:36:11.363586+00:00\",\n
-        \ \"external_trigger\": true,\n  \"last_scheduling_decision\": \"2024-07-29T14:36:30.102789+00:00\",\n
-        \ \"logical_date\": \"2024-07-29T14:36:11.363586+00:00\",\n  \"note\": null,\n
-        \ \"run_type\": \"manual\",\n  \"start_date\": \"2024-07-29T14:36:12.040608+00:00\",\n
+        \"00000000-0000-0000-0000-000000000001\",\n  \"data_interval_end\": \"2024-07-30T08:29:55.039650+00:00\",\n
+        \ \"data_interval_start\": \"2024-07-30T08:29:55.039650+00:00\",\n  \"end_date\":
+        \"2024-07-30T08:30:12.831165+00:00\",\n  \"execution_date\": \"2024-07-30T08:29:55.039650+00:00\",\n
+        \ \"external_trigger\": true,\n  \"last_scheduling_decision\": \"2024-07-30T08:30:12.828982+00:00\",\n
+        \ \"logical_date\": \"2024-07-30T08:29:55.039650+00:00\",\n  \"note\": null,\n
+        \ \"run_type\": \"manual\",\n  \"start_date\": \"2024-07-30T08:29:55.206573+00:00\",\n
         \ \"state\": \"failed\"\n}\n"
     headers:
       Connection:
@@ -73,7 +73,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:31 GMT
+      - Tue, 30 Jul 2024 08:30:15 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -107,7 +107,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Mon, 29 Jul 2024 14:36:31 GMT
+      - Tue, 30 Jul 2024 08:30:15 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -141,7 +141,7 @@ interactions:
       Content-Type:
       - application/problem+json
       Date:
-      - Mon, 29 Jul 2024 14:36:31 GMT
+      - Tue, 30 Jul 2024 08:30:15 GMT
       Server:
       - gunicorn
       X-Robots-Tag:
@@ -173,7 +173,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 29 Jul 2024 14:36:31 GMT
+      - Tue, 30 Jul 2024 08:30:15 GMT
       Server:
       - gunicorn
       X-Robots-Tag: