diff --git a/runner/README.md b/runner/README.md index 0af63dc22..330c6822f 100644 --- a/runner/README.md +++ b/runner/README.md @@ -74,5 +74,40 @@ The `local` backend will submit the run to the locally started shim and runner. ... Shown 3 of 4041 offers, $56.6266 max -Continue? [y/n]: -``` \ No newline at end of file +Continue? [y/n]: +``` + +## Dependencies (WIP) + +These are nonexhaustive lists of external dependencies (executables, libraries) of the `dstack-*` binaries. + +**TODO**: inspect codebase, add missing dependencies. + +### `dstack-shim` + +#### Libraries + +* libc +* ... + +#### Executables + +* `mount` +* `umount` +* `mountpoint` +* `lsblk` +* `mkfs.ext4` +* ... + +Debian/Ubuntu packages: `mount` (`mount`, `umount`), `util-linux` (`mountpoint`, `lsblk`), `e2fsprogs` (`mkfs.ext4`) + +### `dstack-runner` + +#### Libraries + +* libc +* ... + +#### Executables + +* ... diff --git a/runner/internal/shim/docker.go b/runner/internal/shim/docker.go index 856ff2124..293b8179e 100644 --- a/runner/internal/shim/docker.go +++ b/runner/internal/shim/docker.go @@ -106,6 +106,10 @@ func (d *DockerRunner) Run(ctx context.Context, cfg TaskConfig) error { } log.Println("Preparing volumes") + // defer unmountVolumes() before calling prepareVolumes(), as the latter + // may fail when some volumes are already mounted; if the volume is not mounted, + // unmountVolumes() simply skips it + defer func() { _ = unmountVolumes(cfg) }() err = prepareVolumes(cfg) if err != nil { d.state = Pending @@ -274,6 +278,33 @@ func prepareVolumes(taskConfig TaskConfig) error { return nil } +func unmountVolumes(taskConfig TaskConfig) error { + if len(taskConfig.Volumes) == 0 { + return nil + } + log.Println("Unmounting volumes...") + var failed []string + for _, volume := range taskConfig.Volumes { + mountPoint := getVolumeMountPoint(volume.Name) + cmd := exec.Command("mountpoint", mountPoint) + if output, err := cmd.CombinedOutput(); err != nil { + log.Printf("Skipping %s: %s", mountPoint, output) + continue + } + cmd = exec.Command("umount", "-qf", mountPoint) + if output, err := cmd.CombinedOutput(); err != nil { + log.Printf("Failed to unmount %s: %s", mountPoint, output) + failed = append(failed, mountPoint) + } else { + log.Printf("Unmounted: %s\n", mountPoint) + } + } + if len(failed) > 0 { + return fmt.Errorf("Failed to unmount volume(s): %v", failed) + } + return nil +} + func formatAndMountVolume(volume VolumeInfo) error { backend, err := getBackend(volume.Backend) if err != nil {