From 0326e98b2390e780421db83d8e4cf5e9110fdf98 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Tue, 27 Aug 2024 10:22:10 +0200 Subject: [PATCH] restart actions: on restart use data from previous run * ref: cern-sis/issues-inspire/issues/545 --- .../backoffice/workflows/airflow_utils.py | 12 +- backoffice/backoffice/workflows/api/views.py | 10 +- ...kflowViewSet.test_restart_full_dagrun.yaml | 107 ++++++++++-------- .../backoffice/workflows/tests/test_views.py | 12 +- 4 files changed, 83 insertions(+), 58 deletions(-) diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index 5f88b9e7..6f4faf90 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -163,7 +163,7 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): ) -def delete_workflow_dag_runs(workflow_id, workflow_type): +def delete_workflow_dag_runs(workflow_id, workflow_type, params=None): """Deletes runs of a given workflow. :param workflow_id: workflow_id for dags that should be restarted @@ -171,9 +171,17 @@ def delete_workflow_dag_runs(workflow_id, workflow_type): """ executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) - for dag_id in executed_dags_for_workflow: + data = None + + for dag_id, dag_data in executed_dags_for_workflow.items(): delete_workflow_dag(dag_id, str(workflow_id)) + # fetch data from previous drag run + if data is None: + data = dag_data["conf"].get("data") + trigger_airflow_dag( + WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params or data + ) return JsonResponse( data={"message": f"Dag runs for worfklow {workflow_id} have been deleted"} ) diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 17172077..0fe41ca5 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -122,11 +122,12 @@ def create(self, request): WORKFLOW_DAGS[workflow.workflow_type].initialize, workflow.id, ) - return airflow_utils.trigger_airflow_dag( + airflow_utils.trigger_airflow_dag( WORKFLOW_DAGS[workflow.workflow_type].initialize, str(workflow.id), workflow.data, ) + return Response(serializer.data, status=status.HTTP_201_CREATED) @extend_schema( summary="Partially Updates Author", @@ -169,9 +170,14 @@ def resolve(self, request, pk=None): ) utils.add_decision(pk, request.user, serializer.validated_data["value"]) - return airflow_utils.trigger_airflow_dag( + airflow_utils.trigger_airflow_dag( ResolutionDags[serializer.validated_data["value"]].label, pk, extra_data ) + workflow_serializer = self.serializer_class( + get_object_or_404(Workflow, pk=pk) + ) + + return Response(workflow_serializer.data) @extend_schema( summary="Restart an Author Workflow", diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml index 9cfc7cdc..009d219c 100644 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml @@ -1,7 +1,7 @@ interactions: - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' + "00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}' headers: Accept: - '*/*' @@ -10,30 +10,33 @@ interactions: Connection: - keep-alive Content-Length: - - '119' + - '145' Content-Type: - application/json method: POST uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -50,29 +53,30 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -89,16 +93,16 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: @@ -106,7 +110,7 @@ interactions: Content-Type: - application/problem+json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -123,16 +127,16 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: @@ -140,7 +144,7 @@ interactions: Content-Type: - application/problem+json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -159,20 +163,20 @@ interactions: - keep-alive Content-Length: - '0' - Content-Type: - - application/json method: DELETE uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: '' headers: + Cache-Control: + - no-store Connection: - close Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:49 GMT Server: - gunicorn X-Robots-Tag: @@ -182,7 +186,7 @@ interactions: message: NO CONTENT - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' + "00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}' headers: Accept: - '*/*' @@ -191,30 +195,33 @@ interactions: Connection: - keep-alive Content-Length: - - '119' + - '145' Content-Type: - application/json method: POST uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:29.037550+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:29.037550+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:29.037550+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:29.037550+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:49.946419+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:49.946419+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:49.946419+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:49.946419+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:29 GMT + - Wed, 21 Aug 2024 13:20:49 GMT Server: - gunicorn X-Robots-Tag: @@ -233,20 +240,20 @@ interactions: - keep-alive Content-Length: - '0' - Content-Type: - - application/json method: DELETE uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: '' headers: + Cache-Control: + - no-store Connection: - close Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:29 GMT + - Wed, 21 Aug 2024 13:20:56 GMT Server: - gunicorn X-Robots-Tag: diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 319d6c94..46d048de 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -215,6 +215,8 @@ def test_patch_admin(self): "test": "test", }, ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) def test_patch_anonymous(self): self.api_client.force_authenticate(user=self.user) @@ -323,7 +325,7 @@ def setUp(self): super().setUp() self.workflow = Workflow.objects.create( - data={}, + data={"test": "test"}, status="running", core=True, is_update=False, @@ -361,7 +363,8 @@ def test_create_author(self): url = reverse("api:workflows-authors-list") response = self.api_client.post(url, format="json", data=data) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json(), data) @pytest.mark.vcr() def test_accept_author(self): @@ -379,7 +382,6 @@ def test_accept_author(self): self.assertEqual( Decision.objects.filter(workflow=self.workflow.id)[0].action, action ) - airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].approve, self.workflow.id ) @@ -400,6 +402,8 @@ def test_reject_author(self): self.assertEqual( Decision.objects.filter(workflow=self.workflow.id)[0].action, action ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].reject, self.workflow.id @@ -413,8 +417,8 @@ def test_restart_full_dagrun(self): kwargs={"pk": self.workflow.id}, ) response = self.api_client.post(url) - self.assertEqual(response.status_code, 200) + self.assertIn("test", response.json()["conf"]["data"]) @pytest.mark.vcr() def test_restart_a_task(self):