diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py index fb3cff0c..f71fe047 100644 --- a/reana_workflow_controller/config.py +++ b/reana_workflow_controller/config.py @@ -175,3 +175,10 @@ RunStatus.pending, ] """Alive workflow statuses.""" + +REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD = int( + os.getenv("REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD", "120") +) +"""Grace period before terminating the job controller and workflow engine pod. + +The job controller needs to clean up all the running jobs before the end of the grace period.""" diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py index 92589093..1b251265 100644 --- a/reana_workflow_controller/workflow_run_manager.py +++ b/reana_workflow_controller/workflow_run_manager.py @@ -61,6 +61,7 @@ IMAGE_PULL_SECRETS, JOB_CONTROLLER_CONTAINER_PORT, JOB_CONTROLLER_SHUTDOWN_ENDPOINT, + REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD, REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT, REANA_KUBERNETES_JOBS_MEMORY_LIMIT, REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT, @@ -652,6 +653,7 @@ def _create_job_spec( containers=containers, node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL, init_containers=[], + termination_grace_period_seconds=REANA_JOB_CONTROLLER_TERMINATION_GRACE_PERIOD, ) spec.template.spec.service_account_name = ( REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME