diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index 95f85626..29d4e0bc 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -158,9 +158,14 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): """ executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) - for dag_id in executed_dags_for_workflow: + data = None + + for dag_id, dag_data in executed_dags_for_workflow.items(): delete_workflow_dag(dag_id, str(workflow_id)) + # fetch data from previous drag run + if data is None: + data = dag_data["conf"]["data"] return trigger_airflow_dag( - WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params + WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params or data ) diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml index 9cfc7cdc..009d219c 100644 --- a/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml @@ -1,7 +1,7 @@ interactions: - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' + "00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}' headers: Accept: - '*/*' @@ -10,30 +10,33 @@ interactions: Connection: - keep-alive Content-Length: - - '119' + - '145' Content-Type: - application/json method: POST uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -50,29 +53,30 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -89,16 +93,16 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: @@ -106,7 +110,7 @@ interactions: Content-Type: - application/problem+json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -123,16 +127,16 @@ interactions: - gzip, deflate Connection: - keep-alive - Content-Type: - - application/json method: GET uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\": - 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: @@ -140,7 +144,7 @@ interactions: Content-Type: - application/problem+json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:42 GMT Server: - gunicorn X-Robots-Tag: @@ -159,20 +163,20 @@ interactions: - keep-alive Content-Length: - '0' - Content-Type: - - application/json method: DELETE uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: '' headers: + Cache-Control: + - no-store Connection: - close Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:28 GMT + - Wed, 21 Aug 2024 13:20:49 GMT Server: - gunicorn X-Robots-Tag: @@ -182,7 +186,7 @@ interactions: message: NO CONTENT - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000000"}}' + "00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}' headers: Accept: - '*/*' @@ -191,30 +195,33 @@ interactions: Connection: - keep-alive Content-Length: - - '119' + - '145' Content-Type: - application/json method: POST uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns response: body: - string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n - \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": - \"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:29.037550+00:00\",\n - \ \"data_interval_start\": \"2024-07-30T12:14:29.037550+00:00\",\n \"end_date\": - null,\n \"execution_date\": \"2024-07-30T12:14:29.037550+00:00\",\n \"external_trigger\": - true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:29.037550+00:00\",\n - \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n - \ \"state\": \"queued\"\n}\n" + string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n + \ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n + \ \"data_interval_end\": \"2024-08-21T13:20:49.946419+00:00\",\n \"data_interval_start\": + \"2024-08-21T13:20:49.946419+00:00\",\n \"end_date\": null,\n \"execution_date\": + \"2024-08-21T13:20:49.946419+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": + null,\n \"logical_date\": \"2024-08-21T13:20:49.946419+00:00\",\n \"note\": + null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": + \"queued\"\n}\n" headers: + Cache-Control: + - no-store Connection: - close Content-Length: - - '579' + - '621' Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:29 GMT + - Wed, 21 Aug 2024 13:20:49 GMT Server: - gunicorn X-Robots-Tag: @@ -233,20 +240,20 @@ interactions: - keep-alive Content-Length: - '0' - Content-Type: - - application/json method: DELETE uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000 response: body: string: '' headers: + Cache-Control: + - no-store Connection: - close Content-Type: - application/json Date: - - Tue, 30 Jul 2024 12:14:29 GMT + - Wed, 21 Aug 2024 13:20:56 GMT Server: - gunicorn X-Robots-Tag: diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 8775b0b6..1d5c8940 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -279,7 +279,7 @@ def setUp(self): super().setUp() self.workflow = Workflow.objects.create( - data={}, + data={"test": "test"}, status="running", core=True, is_update=False, @@ -361,8 +361,8 @@ def test_restart_full_dagrun(self): kwargs={"pk": self.workflow.id}, ) response = self.api_client.post(url) - self.assertEqual(response.status_code, 200) + self.assertIn("test", response.json()["conf"]["data"]) @pytest.mark.vcr() def test_restart_a_task(self):