Skip to content
This repository has been archived by the owner on Nov 21, 2024. It is now read-only.

Commit

Permalink
restart actions: on restart use data from previous run
Browse files Browse the repository at this point in the history 
* ref: cern-sis/issues-inspire/issues/545
  • Loading branch information
@DonHaul
DonHaul committed Aug 21, 2024
1 parent b80b0de commit 8aade6b
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 54 deletions.
9 changes: 7 additions & 2 deletions backoffice/backoffice/workflows/airflow_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,14 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None):
"""
executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type)

for dag_id in executed_dags_for_workflow:
data = None

for dag_id, dag_data in executed_dags_for_workflow.items():
delete_workflow_dag(dag_id, str(workflow_id))
# fetch data from previous drag run
if data is None:
data = dag_data["conf"]["data"]

return trigger_airflow_dag(
WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params
WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params or data
)
107 changes: 57 additions & 50 deletions ...koffice/workflows/tests/cassettes/TestAuthorWorkflowViewSet.test_restart_full_dagrun.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id":
"00000000-0000-0000-0000-000000000000"}}'
"00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}'
headers:
Accept:
- '*/*'
Expand All @@ -10,30 +10,33 @@ interactions:
Connection:
- keep-alive
Content-Length:
- '119'
- '145'
Content-Type:
- application/json
method: POST
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n
\ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n
\ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\":
\"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n
\ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\":
\"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\":
\"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\":
null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\":
null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\":
\"queued\"\n}\n"
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Length:
- '579'
- '621'
Content-Type:
- application/json
Date:
- Tue, 30 Jul 2024 12:14:28 GMT
- Wed, 21 Aug 2024 13:20:42 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand All @@ -50,29 +53,30 @@ interactions:
- gzip, deflate
Connection:
- keep-alive
Content-Type:
- application/json
method: GET
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:28.743346+00:00\",\n
\ \"data_interval_start\": \"2024-07-30T12:14:28.743346+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-07-30T12:14:28.743346+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:28.743346+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n
\ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\":
\"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n
\ \"data_interval_end\": \"2024-08-21T13:20:42.301248+00:00\",\n \"data_interval_start\":
\"2024-08-21T13:20:42.301248+00:00\",\n \"end_date\": null,\n \"execution_date\":
\"2024-08-21T13:20:42.301248+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\":
null,\n \"logical_date\": \"2024-08-21T13:20:42.301248+00:00\",\n \"note\":
null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\":
\"queued\"\n}\n"
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Length:
- '579'
- '621'
Content-Type:
- application/json
Date:
- Tue, 30 Jul 2024 12:14:28 GMT
- Wed, 21 Aug 2024 13:20:42 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand All @@ -89,24 +93,24 @@ interactions:
- gzip, deflate
Connection:
- keep-alive
Content-Type:
- application/json
method: GET
uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000000
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag'
and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\":
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Length:
- '294'
Content-Type:
- application/problem+json
Date:
- Tue, 30 Jul 2024 12:14:28 GMT
- Wed, 21 Aug 2024 13:20:42 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand All @@ -123,24 +127,24 @@ interactions:
- gzip, deflate
Connection:
- keep-alive
Content-Type:
- application/json
method: GET
uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000000
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag'
and DagRun ID: '00000000-0000-0000-0000-000000000000' not found\",\n \"status\":
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.9.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Length:
- '294'
Content-Type:
- application/problem+json
Date:
- Tue, 30 Jul 2024 12:14:28 GMT
- Wed, 21 Aug 2024 13:20:42 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand All @@ -159,20 +163,20 @@ interactions:
- keep-alive
Content-Length:
- '0'
Content-Type:
- application/json
method: DELETE
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000
response:
body:
string: ''
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Type:
- application/json
Date:
- Tue, 30 Jul 2024 12:14:28 GMT
- Wed, 21 Aug 2024 13:20:49 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand All @@ -182,7 +186,7 @@ interactions:
message: NO CONTENT
- request:
body: '{"dag_run_id": "00000000-0000-0000-0000-000000000000", "conf": {"workflow_id":
"00000000-0000-0000-0000-000000000000"}}'
"00000000-0000-0000-0000-000000000000", "data": {"test": "test"}}}'
headers:
Accept:
- '*/*'
Expand All @@ -191,30 +195,33 @@ interactions:
Connection:
- keep-alive
Content-Length:
- '119'
- '145'
Content-Type:
- application/json
method: POST
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"00000000-0000-0000-0000-000000000000\",\n \"data_interval_end\": \"2024-07-30T12:14:29.037550+00:00\",\n
\ \"data_interval_start\": \"2024-07-30T12:14:29.037550+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-07-30T12:14:29.037550+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-30T12:14:29.037550+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n
\ \"workflow_id\": \"00000000-0000-0000-0000-000000000000\"\n },\n \"dag_id\":
\"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000000\",\n
\ \"data_interval_end\": \"2024-08-21T13:20:49.946419+00:00\",\n \"data_interval_start\":
\"2024-08-21T13:20:49.946419+00:00\",\n \"end_date\": null,\n \"execution_date\":
\"2024-08-21T13:20:49.946419+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\":
null,\n \"logical_date\": \"2024-08-21T13:20:49.946419+00:00\",\n \"note\":
null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\":
\"queued\"\n}\n"
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Length:
- '579'
- '621'
Content-Type:
- application/json
Date:
- Tue, 30 Jul 2024 12:14:29 GMT
- Wed, 21 Aug 2024 13:20:49 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand All @@ -233,20 +240,20 @@ interactions:
- keep-alive
Content-Length:
- '0'
Content-Type:
- application/json
method: DELETE
uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000000
response:
body:
string: ''
headers:
Cache-Control:
- no-store
Connection:
- close
Content-Type:
- application/json
Date:
- Tue, 30 Jul 2024 12:14:29 GMT
- Wed, 21 Aug 2024 13:20:56 GMT
Server:
- gunicorn
X-Robots-Tag:
Expand Down
4 changes: 2 additions & 2 deletions backoffice/backoffice/workflows/tests/test_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ def setUp(self):
super().setUp()

self.workflow = Workflow.objects.create(
data={},
data={"test": "test"},
status="running",
core=True,
is_update=False,
Expand Down Expand Up @@ -361,8 +361,8 @@ def test_restart_full_dagrun(self):
kwargs={"pk": self.workflow.id},
)
response = self.api_client.post(url)

self.assertEqual(response.status_code, 200)
self.assertIn("test", response.json()["conf"]["data"])

@pytest.mark.vcr()
def test_restart_a_task(self):
Expand Down

0 comments on commit 8aade6b

Please sign in to comment.