[shim] Unmount volumes when run exits (#1937)

Fixes: #1936

runner/README.md

@@ -74,5 +74,40 @@ The `local` backend will submit the run to the locally started shim and runner.
     ...                                                                              
  Shown 3 of 4041 offers, $56.6266 max
 
-Continue? [y/n]: 
-```
+Continue? [y/n]:
+```
+
+## Dependencies (WIP)
+
+These are nonexhaustive lists of external dependencies (executables, libraries) of the `dstack-*` binaries.
+
+**TODO**: inspect codebase, add missing dependencies.
+
+### `dstack-shim`
+
+#### Libraries
+
+* libc
+* ...
+
+#### Executables
+
+* `mount`
+* `umount`
+* `mountpoint`
+* `lsblk`
+* `mkfs.ext4`
+* ...
+
+Debian/Ubuntu packages: `mount` (`mount`, `umount`), `util-linux` (`mountpoint`, `lsblk`), `e2fsprogs` (`mkfs.ext4`)
+
+### `dstack-runner`
+
+#### Libraries
+
+* libc
+* ...
+
+#### Executables
+
+* ...

runner/internal/shim/docker.go

@@ -106,6 +106,10 @@ func (d *DockerRunner) Run(ctx context.Context, cfg TaskConfig) error {
 	}
 
 	log.Println("Preparing volumes")
+	// defer unmountVolumes() before calling prepareVolumes(), as the latter
+	// may fail when some volumes are already mounted; if the volume is not mounted,
+	// unmountVolumes() simply skips it
+	defer func() { _ = unmountVolumes(cfg) }()
 	err = prepareVolumes(cfg)
 	if err != nil {
 		d.state = Pending
@@ -274,6 +278,33 @@ func prepareVolumes(taskConfig TaskConfig) error {
 	return nil
 }
 
+func unmountVolumes(taskConfig TaskConfig) error {
+	if len(taskConfig.Volumes) == 0 {
+		return nil
+	}
+	log.Println("Unmounting volumes...")
+	var failed []string
+	for _, volume := range taskConfig.Volumes {
+		mountPoint := getVolumeMountPoint(volume.Name)
+		cmd := exec.Command("mountpoint", mountPoint)
+		if output, err := cmd.CombinedOutput(); err != nil {
+			log.Printf("Skipping %s: %s", mountPoint, output)
+			continue
+		}
+		cmd = exec.Command("umount", "-qf", mountPoint)
+		if output, err := cmd.CombinedOutput(); err != nil {
+			log.Printf("Failed to unmount %s: %s", mountPoint, output)
+			failed = append(failed, mountPoint)
+		} else {
+			log.Printf("Unmounted: %s\n", mountPoint)
+		}
+	}
+	if len(failed) > 0 {
+		return fmt.Errorf("Failed to unmount volume(s): %v", failed)
+	}
+	return nil
+}
+
 func formatAndMountVolume(volume VolumeInfo) error {
 	backend, err := getBackend(volume.Backend)
 	if err != nil {