safer srun termination

src/radical/pilot/agent/executing/popen.py

@@ -317,9 +317,17 @@ def _check_running(self, to_watch, to_cancel):
                     # process group (which should include the actual launch
                     # method)
                     try:
-                        # kill the whole process group
+                        # kill the whole process group.
+                        # Try SIGINT first to allow signal handlers, then
+                        # SIGTERM to allow clean termination, then SIGKILL to
+                        # enforce termination.
                         pgrp = os.getpgid(task['proc'].pid)
+                        os.killpg(pgrp, signal.SIGINT)
+                        time.sleep(0.1)
+                        os.killpg(pgrp, signal.SIGTERM)
+                        time.sleep(0.1)
                         os.killpg(pgrp, signal.SIGKILL)
+
                     except OSError:
                         # lost race: task is already gone, we ignore this
                         # FIXME: collect and move to DONE/FAILED

src/radical/pilot/agent/launch_method/srun.py

@@ -150,8 +150,14 @@ def get_launch_cmds(self, task, exec_path):
                 gpus_per_task = len(slots[0]['gpus'])
 
         mapping = ''
-        if n_tasks > 1 and td['use_mpi'] is False:
-            mapping += '--kill-on-bad-exit=0 '
+        if n_tasks > 1:
+            if td['use_mpi'] is False:
+                mapping += '-K0 '  # '--kill-on-bad-exit=0 '
+            else:
+                # ensure that all ranks are killed if one rank fails
+                mapping += '-K1 '  # '--kill-on-bad-exit=1 '
+                # allow step cancellation with single SIGINT
+                mapping += '--quit-on-interrupt '
 
         if self._exact:
             mapping += '--exact '