feat(manager): increase termination period of run-batch pods (reanahu…

…b#572)
mdonadoni · Feb 23, 2024 · 6873ea3 · 6873ea3
1 parent 08ab9a3
commit 6873ea3
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py
@@ -175,3 +175,10 @@
     RunStatus.pending,
 ]
 """Alive workflow statuses."""
+
+REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD = int(
+    os.getenv("REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD", "120")
+)
+"""Grace period before terminating the job controller and workflow engine pod.
+
+The job controller needs to clean up all the running jobs before the end of the grace period."""
diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py
@@ -61,6 +61,7 @@
     IMAGE_PULL_SECRETS,
     JOB_CONTROLLER_CONTAINER_PORT,
     JOB_CONTROLLER_SHUTDOWN_ENDPOINT,
+    REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD,
     REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT,
     REANA_KUBERNETES_JOBS_MEMORY_LIMIT,
     REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT,
@@ -652,6 +653,7 @@ def _create_job_spec(
             containers=containers,
             node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL,
             init_containers=[],
+            termination_grace_period_seconds=REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD,
         )
         spec.template.spec.service_account_name = (
             REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME