diff --git a/pkg/common/func.go b/pkg/common/func.go index 3ba8d84..4b442c4 100644 --- a/pkg/common/func.go +++ b/pkg/common/func.go @@ -50,10 +50,10 @@ func NewInterLinkConfig() (InterLinkConfig, error) { return InterLinkConfig{}, err } - log.G(context.Background()).Info("Loading InterLink config from " + path) + log.G(context.Background()).Info("\u2705 Loading InterLink config from " + path) yfile, err := os.ReadFile(path) if err != nil { - log.G(context.Background()).Error("Error opening config file, exiting...") + log.G(context.Background()).Error("\u274C Error opening config file, exiting...") return InterLinkConfig{}, err } yaml.Unmarshal(yfile, &InterLinkConfigInst) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index e836a64..cfb8d2a 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -7,357 +7,484 @@ import ( "os" "strconv" "strings" + "time" exec "github.com/alexellis/go-execute/pkg/v1" "github.com/containerd/containerd/log" v1 "k8s.io/api/core/v1" - "time" + "errors" commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" -) -// CreateHandler creates a Docker Container based on data provided by the InterLink API. -func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { - log.G(h.Ctx).Info("Docker Sidecar: received Create call") - var execReturn exec.ExecResult - statusCode := http.StatusOK - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, nil) - return - } + OSexec "os/exec" + "path/filepath" +) - var req []commonIL.RetrievedPodData - err = json.Unmarshal(bodyBytes, &req) +type DockerRunStruct struct { + Name string `json:"name"` + Command string `json:"command"` + IsInitContainer bool `json:"isInitContainer"` + GpuArgs string `json:"gpuArgs"` +} - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, nil) - return - } +func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w http.ResponseWriter) ([]DockerRunStruct, error) { - for _, data := range req { + var dockerRunStructs []DockerRunStruct + var gpuArgs string = "" - podUID := string(data.Pod.UID) - podNamespace := string(data.Pod.Namespace) + podUID := string(podData.Pod.UID) + podNamespace := string(podData.Pod.Namespace) - pathsOfVolumes := make(map[string]string) + pathsOfVolumes := make(map[string]string) - for _, volume := range data.Pod.Spec.Volumes { - if volume.HostPath != nil { - if *volume.HostPath.Type == v1.HostPathDirectoryOrCreate || *volume.HostPath.Type == v1.HostPathDirectory { - _, err := os.Stat(volume.HostPath.Path) - if *volume.HostPath.Type == v1.HostPathDirectory { - if os.IsNotExist(err) { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } - pathsOfVolumes[volume.Name] = volume.HostPath.Path - } else if *volume.HostPath.Type == v1.HostPathDirectoryOrCreate { - if os.IsNotExist(err) { - err = os.MkdirAll(volume.HostPath.Path, os.ModePerm) - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } else { - pathsOfVolumes[volume.Name] = volume.HostPath.Path - } + for _, volume := range podData.Pod.Spec.Volumes { + if volume.HostPath != nil { + if *volume.HostPath.Type == v1.HostPathDirectoryOrCreate || *volume.HostPath.Type == v1.HostPathDirectory { + _, err := os.Stat(volume.HostPath.Path) + if *volume.HostPath.Type == v1.HostPathDirectory { + if os.IsNotExist(err) { + HandleErrorAndRemoveData(h, w, "Host path directory does not exist", err, podNamespace, podUID) + return dockerRunStructs, errors.New("Host path directory does not exist") + } + pathsOfVolumes[volume.Name] = volume.HostPath.Path + } else if *volume.HostPath.Type == v1.HostPathDirectoryOrCreate { + if os.IsNotExist(err) { + err = os.MkdirAll(volume.HostPath.Path, os.ModePerm) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during mkdir of host path directory", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during mkdir of host path directory") } else { pathsOfVolumes[volume.Name] = volume.HostPath.Path } + } else { + pathsOfVolumes[volume.Name] = volume.HostPath.Path } } } + } - if volume.PersistentVolumeClaim != nil { - if _, ok := pathsOfVolumes[volume.PersistentVolumeClaim.ClaimName]; !ok { - // WIP: This is a temporary solution to mount CVMFS volumes - pathsOfVolumes[volume.PersistentVolumeClaim.ClaimName] = "/mnt/cvmfs" - } - + if volume.PersistentVolumeClaim != nil { + if _, ok := pathsOfVolumes[volume.PersistentVolumeClaim.ClaimName]; !ok { + // WIP: this is a temporary solution to mount CVMFS volumes for persistent volume claims case + pathsOfVolumes[volume.PersistentVolumeClaim.ClaimName] = "/cvmfs" } - } - allContainers := map[string][]v1.Container{ - "initContainers": data.Pod.Spec.InitContainers, - "containers": data.Pod.Spec.Containers, } + } - // iterate over all containers - for containerType, containers := range allContainers { - isInitContainer := containerType == "initContainers" + allContainers := map[string][]v1.Container{ + "initContainers": podData.Pod.Spec.InitContainers, + "containers": podData.Pod.Spec.Containers, + } - for _, container := range containers { + for containerType, containers := range allContainers { + isInitContainer := containerType == "initContainers" - containerName := podNamespace + "-" + podUID + "-" + container.Name + for _, container := range containers { - var isGpuRequested bool = false - var additionalGpuArgs []string + containerName := podNamespace + "-" + podUID + "-" + container.Name - if val, ok := container.Resources.Limits["nvidia.com/gpu"]; ok { + var isGpuRequested bool = false + var additionalGpuArgs []string - numGpusRequested := val.Value() + if val, ok := container.Resources.Limits["nvidia.com/gpu"]; ok { - // if the container is requesting 0 GPU, skip the GPU assignment - if numGpusRequested == 0 { - log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU") - } else { + numGpusRequested := val.Value() + + // if the container is requesting 0 GPU, skip the GPU assignment + if numGpusRequested == 0 { + log.G(h.Ctx).Info("\u2705 Container " + containerName + " is not requesting a GPU") + } else { - log.G(h.Ctx).Info("Container " + containerName + " is requesting " + val.String() + " GPU") + log.G(h.Ctx).Info("\u2705 Container " + containerName + " is requesting " + val.String() + " GPU") - isGpuRequested = true + isGpuRequested = true - numGpusRequestedInt := int(numGpusRequested) - _, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt) + numGpusRequestedInt := int(numGpusRequested) + _, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt) - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during request of get available GPUs", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during request of get available GPUs") + } - gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName) - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } + gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during request of get and assign of an available GPU", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during request of get and assign of an available GPU") + } - var gpuUUIDs string = "" - for _, gpuSpec := range gpuSpecs { - if gpuSpec.UUID == gpuSpecs[len(gpuSpecs)-1].UUID { - gpuUUIDs += strconv.Itoa(gpuSpec.Index) - } else { - gpuUUIDs += strconv.Itoa(gpuSpec.Index) + "," - } + var gpuUUIDs string = "" + for _, gpuSpec := range gpuSpecs { + if gpuSpec.UUID == gpuSpecs[len(gpuSpecs)-1].UUID { + gpuUUIDs += strconv.Itoa(gpuSpec.Index) + } else { + gpuUUIDs += strconv.Itoa(gpuSpec.Index) + "," } - - additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs) } - } else { - log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU") + additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs) + gpuArgs = "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=" + gpuUUIDs } - var envVars string = "" - // add environment variables to the docker command - for _, envVar := range container.Env { - if envVar.Value != "" { - // check if the env variable is an array, in this case the value needs to be between '' - if strings.Contains(envVar.Value, "[") { - envVars += " -e " + envVar.Name + "='" + envVar.Value + "'" - } else { - envVars += " -e " + envVar.Name + "=" + envVar.Value - } + } + + var envVars string = "" + for _, envVar := range container.Env { + if envVar.Value != "" { + if strings.Contains(envVar.Value, "[") { + envVars += " -e " + envVar.Name + "='" + envVar.Value + "'" } else { - envVars += " -e " + envVar.Name + envVars += " -e " + envVar.Name + "=" + envVar.Value } + } else { + envVars += " -e " + envVar.Name } + } - // iterate over the container volumes and mount them in the docker command line; get the volume path in the host from pathsOfVolumes - for _, volumeMount := range container.VolumeMounts { - if volumeMount.MountPath != "" { + for _, volumeMount := range container.VolumeMounts { + if volumeMount.MountPath != "" { - // check if volumeMount.name is inside pathsOfVolumes, if it is add the volume to the docker command - if _, ok := pathsOfVolumes[volumeMount.Name]; !ok { - log.G(h.Ctx).Error("Volume " + volumeMount.Name + " not found in pathsOfVolumes") - continue - } - if volumeMount.ReadOnly { - envVars += " -v " + pathsOfVolumes[volumeMount.Name] + ":" + volumeMount.MountPath + ":ro" + if _, ok := pathsOfVolumes[volumeMount.Name]; !ok { + continue + } + if volumeMount.ReadOnly { + envVars += " -v " + pathsOfVolumes[volumeMount.Name] + ":" + volumeMount.MountPath + ":ro" + } else { + if volumeMount.MountPropagation != nil && *volumeMount.MountPropagation == v1.MountPropagationBidirectional { + envVars += " -v " + pathsOfVolumes[volumeMount.Name] + ":" + volumeMount.MountPath + ":shared" } else { - // if it is Bidirectional, add :shared to the volume - if volumeMount.MountPropagation != nil && *volumeMount.MountPropagation == v1.MountPropagationBidirectional { - envVars += " -v " + pathsOfVolumes[volumeMount.Name] + ":" + volumeMount.MountPath + ":shared" - } else { - envVars += " -v " + pathsOfVolumes[volumeMount.Name] + ":" + volumeMount.MountPath - } + envVars += " -v " + pathsOfVolumes[volumeMount.Name] + ":" + volumeMount.MountPath } } } + } - //docker run --privileged -v /home:/home -d --name demo1 docker:dind + envVars += " --network=host" + cmd := []string{"run", "-d", "--name", containerName} - log.G(h.Ctx).Info("- Creating container " + containerName) + cmd = append(cmd, envVars) - cmd := []string{"run", "-d", "--name", containerName} + if container.SecurityContext != nil && container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged { + cmd = append(cmd, "--privileged") + } - cmd = append(cmd, envVars) + if isGpuRequested { + cmd = append(cmd, additionalGpuArgs...) + } - if container.SecurityContext != nil && container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged { - cmd = append(cmd, "--privileged") - //cmd = append(cmd, "--cap-add=SYS_ADMIN") - //cmd = append(cmd, "--device=/dev/fuse") - //cmd = append(cmd, "--security-opt=apparmor:unconfined") - } + var additionalPortArgs []string - if isGpuRequested { - cmd = append(cmd, additionalGpuArgs...) + for _, port := range container.Ports { + if port.HostPort != 0 { + additionalPortArgs = append(additionalPortArgs, "-p", strconv.Itoa(int(port.HostPort))+":"+strconv.Itoa(int(port.ContainerPort))) } + } - var additionalPortArgs []string + cmd = append(cmd, additionalPortArgs...) - for _, port := range container.Ports { - if port.HostPort != 0 { - additionalPortArgs = append(additionalPortArgs, "-p", strconv.Itoa(int(port.HostPort))+":"+strconv.Itoa(int(port.ContainerPort))) - } - } + mounts, err := prepareMounts(h.Ctx, h.Config, podData, container) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during preparing mounts for the POD", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during preparing mounts for the POD") + } + + cmd = append(cmd, mounts) + + memoryLimitsArray := []string{} + cpuLimitsArray := []string{} + + if container.Resources.Limits.Memory().Value() != 0 { + memoryLimitsArray = append(memoryLimitsArray, "--memory", strconv.Itoa(int(container.Resources.Limits.Memory().Value()))+"b") + } + if container.Resources.Limits.Cpu().Value() != 0 { + cpuLimitsArray = append(cpuLimitsArray, "--cpus", strconv.FormatFloat(float64(container.Resources.Limits.Cpu().Value()), 'f', -1, 64)) + } + + cmd = append(cmd, memoryLimitsArray...) + cmd = append(cmd, cpuLimitsArray...) - cmd = append(cmd, additionalPortArgs...) + containerCommands := []string{} + containerArgs := []string{} + mountFileCommand := []string{} - //if h.Config.ExportPodData { - mounts, err := prepareMounts(h.Ctx, h.Config, req, container) + // if container has a command and args, call parseContainerCommandAndReturnArgs + if len(container.Command) > 0 || len(container.Args) > 0 { + mountFileCommand, containerCommands, containerArgs, err = parseContainerCommandAndReturnArgs(h.Ctx, h.Config, podUID, podNamespace, container) if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return + HandleErrorAndRemoveData(h, w, "An error occurred during the parse of the container commands and arguments", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during the parse of the container commands and arguments") } - // print the mounts for debugging - log.G(h.Ctx).Info("Mounts: " + mounts) + cmd = append(cmd, mountFileCommand...) + } - cmd = append(cmd, mounts) - //} + cmd = append(cmd, container.Image) + cmd = append(cmd, containerCommands...) + cmd = append(cmd, containerArgs...) - memoryLimitsArray := []string{} - cpuLimitsArray := []string{} + dockerOptions := "" - if container.Resources.Limits.Memory().Value() != 0 { - memoryLimitsArray = append(memoryLimitsArray, "--memory", strconv.Itoa(int(container.Resources.Limits.Memory().Value()))+"b") - } - if container.Resources.Limits.Cpu().Value() != 0 { - cpuLimitsArray = append(cpuLimitsArray, "--cpus", strconv.FormatFloat(float64(container.Resources.Limits.Cpu().Value()), 'f', -1, 64)) + if dockerFlags, ok := podData.Pod.ObjectMeta.Annotations["docker-options.vk.io/flags"]; ok { + parsedDockerOptions := strings.Split(dockerFlags, " ") + for _, option := range parsedDockerOptions { + dockerOptions += " " + option } + } - cmd = append(cmd, memoryLimitsArray...) - cmd = append(cmd, cpuLimitsArray...) + shell := exec.ExecTask{ + Command: "docker" + dockerOptions, + Args: cmd, + Shell: true, + } - containerCommands := []string{} - containerArgs := []string{} - mountFileCommand := []string{} + dockerRunStructs = append(dockerRunStructs, DockerRunStruct{ + Name: containerName, + Command: "docker " + strings.Join(shell.Args, " "), + IsInitContainer: isInitContainer, + GpuArgs: gpuArgs, + }) + } + } - // if container has a command and args, call parseContainerCommandAndReturnArgs - if len(container.Command) > 0 || len(container.Args) > 0 { - log.G(h.Ctx).Info("Container has command and args defined. Parsing...") - log.G(h.Ctx).Info("Container command: " + strings.Join(container.Command, " ")) - log.G(h.Ctx).Info("Container args: " + strings.Join(container.Args, " ")) + return dockerRunStructs, nil +} - mountFileCommand, containerCommands, containerArgs, err = parseContainerCommandAndReturnArgs(h.Ctx, h.Config, req, container) - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } - cmd = append(cmd, mountFileCommand...) - } +func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { - // log container commands and args - log.G(h.Ctx).Info("Container commands: " + strings.Join(containerCommands, " ")) - log.G(h.Ctx).Info("Container args: " + strings.Join(containerArgs, " ")) + log.G(h.Ctx).Info("\u23F3 [CREATE CALL] Received create call from InterLink ") - cmd = append(cmd, container.Image) - cmd = append(cmd, containerCommands...) - cmd = append(cmd, containerArgs...) + var execReturn exec.ExecResult + statusCode := http.StatusOK - dockerOptions := "" + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during read of body request for pod creation", err, "", "") + return + } - if dockerFlags, ok := data.Pod.ObjectMeta.Annotations["docker-options.vk.io/flags"]; ok { - parsedDockerOptions := strings.Split(dockerFlags, " ") - for _, option := range parsedDockerOptions { - dockerOptions += " " + option - } - } + var req []commonIL.RetrievedPodData + err = json.Unmarshal(bodyBytes, &req) - shell := exec.ExecTask{ - Command: "docker" + dockerOptions, - Args: cmd, - Shell: true, - } + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during json unmarshal of data from pod creation request", err, "", "") + return + } - log.G(h.Ctx).Info("Docker command: " + strings.Join(shell.Args, " ")) + wd, err := os.Getwd() + if err != nil { + HandleErrorAndRemoveData(h, w, "Unable to get current working directory", err, "", "") + return + } - execReturn, err = shell.Execute() - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Request data unmarshalled successfully and current working directory detected") - if execReturn.Stdout == "" { - eval := "Conflict. The container name \"/" + containerName + "\" is already in use" - if strings.Contains(execReturn.Stderr, eval) { - log.G(h.Ctx).Warning("Container named " + containerName + " already exists. Skipping its creation.") - } else { - log.G(h.Ctx).Error("Unable to create container " + containerName + " : " + execReturn.Stderr) - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return + for _, data := range req { + + podUID := string(data.Pod.UID) + podNamespace := string(data.Pod.Namespace) + + podDirectoryPath := filepath.Join(wd, h.Config.DataRootFolder+podNamespace+"-"+podUID) + + // call prepareDockerRuns to get the DockerRunStruct array + dockerRunStructs, err := h.prepareDockerRuns(data, w) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during preparing of docker run commmands", err, "", "") + return + } + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Docker run commands prepared successfully") + + // from dockerRunStructs, create two arrays: one for initContainers and one for containers + var initContainers []DockerRunStruct + var containers []DockerRunStruct + var gpuArgs string + + for _, dockerRunStruct := range dockerRunStructs { + if dockerRunStruct.IsInitContainer { + initContainers = append(initContainers, dockerRunStruct) + } else { + containers = append(containers, dockerRunStruct) + } + } + + // check if between the containers there is a container that requires a GPU + for _, container := range containers { + if container.GpuArgs != "" { + gpuArgs = container.GpuArgs + } + } + + gpuArgsAsArray := []string{} + if gpuArgs != "" { + gpuArgsAsArray = strings.Split(gpuArgs, " ") + } + + dindImage := "ghcr.io/extrality/nvidia-dind" + if gpuArgs == "" { + dindImage = "docker:dind" + } + + dindContainerArgs := []string{"run"} + dindContainerArgs = append(dindContainerArgs, gpuArgsAsArray...) + if _, err := os.Stat("/cvmfs"); err == nil { + dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") + } + + dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", "/home:/home", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", string(data.Pod.UID)+"_dind", dindImage) + + var dindContainerID string + shell := exec.ExecTask{ + Command: "docker", + Args: dindContainerArgs, + Shell: true, + } + + execReturn, err = shell.Execute() + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the execution of DIND container command", err, "", "") + return + } + dindContainerID = execReturn.Stdout + + log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container created successfully with ID: " + dindContainerID) + + // create a variable of maximum number of retries + maxRetries := 10 + + // wait until the dind container is up and running by check that the command docker ps inside of it does not return an error + for { + + if maxRetries == 0 { + HandleErrorAndRemoveData(h, w, "The number of attempts to check if the DIND container is running is 0. This means that an error occurred during the creation of the DIND container", err, "", "") + return + } + + cmd := OSexec.Command("docker", "logs", string(data.Pod.UID)+"_dind") + output, err := cmd.CombinedOutput() + + if err != nil { + time.Sleep(1 * time.Second) + } + + if strings.Contains(string(output), "API listen on /var/run/docker.sock") { + break + } else { + time.Sleep(1 * time.Second) + } + + maxRetries -= 1 + + } + + log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container is up and running, ready to create the containers inside of it") + + if len(initContainers) > 0 { + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Start creating init containers") + + initContainersCommand := "#!/bin/sh\n" + for _, initContainer := range initContainers { + initContainersCommand += initContainer.Command + "\n" + } + err = os.WriteFile(podDirectoryPath+"/init_containers_command.sh", []byte(initContainersCommand), 0644) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the init container script file", err, "", "") + return + } + + shell = exec.ExecTask{ + Command: "docker", + Args: []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/init_containers_command.sh"}, + } + + _, err := shell.Execute() + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the exec of the init container command script ", err, "", "") + return + } + // Poll the container status until it exits + for { + + allInitContainersCompleted := false + initContainersCompleted := 0 + + for _, initContainer := range initContainers { + + shell = exec.ExecTask{ + Command: "docker", + Args: []string{"exec", string(data.Pod.UID) + "_dind", "docker", "inspect", "--format='{{.State.Status}}'", initContainer.Name}, } - } else { - log.G(h.Ctx).Info("-- Created container " + containerName) - } - shell = exec.ExecTask{ - Command: "docker", - Args: []string{"ps", "-aqf", "name=^" + containerName + "$"}, - Shell: true, - } + statusReturn, err := shell.Execute() + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during inspect of init container", err, "", "") + return + } - execReturn, err = shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") - if execReturn.Stderr != "" { - log.G(h.Ctx).Error("Failed to retrieve " + containerName + " ID : " + execReturn.Stderr) - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } else if execReturn.Stdout == "" { - log.G(h.Ctx).Error("Container name not found. Maybe creation failed?") - } else { - containerID := execReturn.Stdout - - log.G(h.Ctx).Debug("-- Retrieved " + containerName + " ID: " + execReturn.Stdout) - - if isInitContainer { - log.G(h.Ctx).Info("Waiting for Init Container " + containerName + " to complete") - - // Poll the container status until it exits - for { - statusShell := exec.ExecTask{ - Command: "docker", - Args: []string{"inspect", "--format='{{.State.Status}}'", containerID}, - Shell: true, - } - - statusReturn, err := statusShell.Execute() - if err != nil { - log.G(h.Ctx).Error("Failed to inspect Init Container " + containerName + " : " + err.Error()) - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while inspecting container. Check Docker Sidecar's logs", err, &data) - return - } - - status := strings.Trim(statusReturn.Stdout, "'\n") - if status == "exited" { - log.G(h.Ctx).Info("Init Container " + containerName + " has completed") - break - } else { - time.Sleep(1 * time.Second) // Wait for a second before polling again - } - } + status := strings.Trim(statusReturn.Stdout, "'\n") + if status == "exited" { + initContainersCompleted += 1 + } else { + time.Sleep(1 * time.Second) // Wait for a second before polling again } } + if initContainersCompleted == len(initContainers) { + allInitContainersCompleted = true + } + if allInitContainersCompleted { + break + } } + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Init containers created and executed successfully") } - } - w.WriteHeader(statusCode) + // create a file called containers_command.sh and write the containers commands to it, use WriteFile function + containersCommand := "#!/bin/sh\n" + for _, container := range containers { + containersCommand += container.Command + "\n" + } + err = os.WriteFile(podDirectoryPath+"/containers_command.sh", []byte(containersCommand), 0644) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the container commands script.", err, "", "") + return + } + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Containers commands written to the script file") - if statusCode != http.StatusOK { - w.Write([]byte("Some errors occurred while creating containers. Check Docker Sidecar's logs")) - } else { - w.Write([]byte("Containers created")) + shell = exec.ExecTask{ + Command: "docker", + Args: []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/containers_command.sh"}, + } + + execReturn, err = shell.Execute() + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the execution of the container command script", err, "", "") + return + } + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Containers created successfully") + + w.WriteHeader(statusCode) + + if statusCode != http.StatusOK { + w.Write([]byte("Some errors occurred while creating containers. Check Docker Sidecar's logs")) + } else { + w.Write([]byte("Containers created")) + } } + } -func HandleErrorAndRemoveData(h *SidecarHandler, w http.ResponseWriter, statusCode int, s string, err error, data *commonIL.RetrievedPodData) { - statusCode = http.StatusInternalServerError +func HandleErrorAndRemoveData(h *SidecarHandler, w http.ResponseWriter, s string, err error, podNamespace string, podUID string) { log.G(h.Ctx).Error(err) - w.WriteHeader(statusCode) + log.G(h.Ctx).Info("\u274C Error description: " + s) + w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("Some errors occurred while creating container. Check Docker Sidecar's logs")) - if data != nil { - os.RemoveAll(h.Config.DataRootFolder + data.Pod.Namespace + "-" + string(data.Pod.UID)) + if podNamespace != "" && podUID != "" { + os.RemoveAll(h.Config.DataRootFolder + podNamespace + "-" + podUID) } } diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index 09feb83..c7f0fb9 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -14,7 +14,7 @@ import ( // DeleteHandler stops and deletes Docker containers from provided data func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { - log.G(h.Ctx).Info("Docker Sidecar: received Delete call") + log.G(h.Ctx).Info("\u23F3 [DELETE CALL] Received delete call from Interlink") var execReturn exec.ExecResult statusCode := http.StatusOK bodyBytes, err := io.ReadAll(r.Body) @@ -40,15 +40,14 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { podUID := string(pod.UID) podNamespace := string(pod.Namespace) - for _, container := range pod.Spec.Containers { containerName := podNamespace + "-" + podUID + "-" + container.Name - log.G(h.Ctx).Debug("- Deleting container " + containerName) + log.G(h.Ctx).Debug("\u2705 [DELETE CALL] Deleting container " + containerName) // added a timeout to the stop container command - cmd := []string{"stop", "-t", "10", containerName} + cmd := []string{"exec", podUID + "_dind", "docker", "stop", "-t", "10", containerName} shell := exec.ExecTask{ Command: "docker", Args: cmd, @@ -58,9 +57,9 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { if execReturn.Stderr != "" { if strings.Contains(execReturn.Stderr, "No such container") { - log.G(h.Ctx).Debug("-- Unable to find container " + containerName + ". Probably already removed? Skipping its removal") + log.G(h.Ctx).Debug("\u26A0 [DELETE CALL] Unable to find container " + containerName + ". Probably already removed? Skipping its removal") } else { - log.G(h.Ctx).Error("-- Error stopping container " + containerName + ". Skipping its removal") + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error stopping container " + containerName + ". Skipping its removal") statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) @@ -70,7 +69,7 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { } if execReturn.Stdout != "" { - cmd = []string{"rm", execReturn.Stdout} + cmd = []string{"exec", podUID + "_dind", "docker", "rm", execReturn.Stdout} shell = exec.ExecTask{ Command: "docker", Args: cmd, @@ -80,16 +79,35 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") if execReturn.Stderr != "" { - log.G(h.Ctx).Error("-- Error deleting container " + containerName) + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting container " + containerName) statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) return } else { - log.G(h.Ctx).Info("- Deleted container " + containerName) + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted container " + containerName) } } + cmd = []string{"rm", "-f", podUID + "_dind"} + shell = exec.ExecTask{ + Command: "docker", + Args: cmd, + Shell: true, + } + execReturn, _ = shell.Execute() + execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") + + if execReturn.Stderr != "" { + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting container " + podUID + "_dind") + statusCode = http.StatusInternalServerError + w.WriteHeader(statusCode) + w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) + return + } else { + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted container " + podUID + "_dind") + } + // check if the container has GPU devices attacched using the GpuManager and release them h.GpuManager.Release(containerName) diff --git a/pkg/docker/GetLogs.go b/pkg/docker/GetLogs.go index 643241d..e3837da 100644 --- a/pkg/docker/GetLogs.go +++ b/pkg/docker/GetLogs.go @@ -16,16 +16,11 @@ import ( // GetLogsHandler performs a Docker logs command and returns its manipulated output func (h *SidecarHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request) { - log.G(h.Ctx).Info("Docker Sidecar: received GetLogs call") + log.G(h.Ctx).Info("\u23F3 [LOGS CALL]: received get logs call") var req commonIL.LogStruct statusCode := http.StatusOK currentTime := time.Now() - //orario, _ := time.Parse("2006-01-02T15:04:05.999999999Z", "2023-09-14T10:35:44.665672258Z") - //test := commonIL.LogStruct{PodName: "test-pod", ContainerName: "busyecho", Opts: commonIL.ContainerLogOpts{Tail: 0, LimitBytes: 350, SinceTime: orario, Timestamps: true}} - //testBytes, _ := json.Marshal(test) - //log.G(h.Ctx).Debug(string(testBytes)) - bodyBytes, err := io.ReadAll(r.Body) if err != nil { statusCode = http.StatusInternalServerError @@ -49,10 +44,7 @@ func (h *SidecarHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request) containerName := podNamespace + "-" + podUID + "-" + req.ContainerName - //var cmd *OSexec.Cmd - // here check if the container exists, if not returm empty logs, exec docker ps and check if the container is listed in the output, if not return - // http.StatusOk and empty logs - cmd := OSexec.Command("docker", "ps", "-a", "--format", "{{.Names}}") + cmd := OSexec.Command("docker", "exec", podUID+"_dind", "docker", "ps", "-a", "--format", "{{.Names}}") output, err := cmd.CombinedOutput() if err != nil { log.G(h.Ctx).Error(err) @@ -74,13 +66,12 @@ func (h *SidecarHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request) w.Write([]byte("No logs available for container " + containerName + ". Container not found.")) return } - //var cmd *OSexec.Cmd if req.Opts.Timestamps { - cmd = OSexec.Command("docker", "logs", "-t", containerName) + cmd = OSexec.Command("docker", "exec", podUID+"_dind", "docker", "logs", "-t", containerName) } else { - cmd = OSexec.Command("docker", "logs", containerName) + cmd = OSexec.Command("docker", "exec", podUID+"_dind", "docker", "logs", containerName) } output, err = cmd.CombinedOutput() diff --git a/pkg/docker/Status.go b/pkg/docker/Status.go index c93fd8a..4e002f9 100644 --- a/pkg/docker/Status.go +++ b/pkg/docker/Status.go @@ -16,7 +16,7 @@ import ( // StatusHandler checks Docker Container's status by running docker ps -af command and returns that status func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { - log.G(h.Ctx).Info("Docker Sidecar: received GetStatus call") + log.G(h.Ctx).Info("\u23F3 [STATUS CALL] received get status call") var resp []commonIL.PodStatus var req []*v1.Pod statusCode := http.StatusOK @@ -47,9 +47,7 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace}) for _, container := range pod.Spec.Containers { containerName := podNamespace + "-" + podUID + "-" + container.Name - - log.G(h.Ctx).Debug("- Getting status for container " + containerName) - cmd := []string{"ps -af name=^" + containerName + "$ --format \"{{.Status}}\""} + cmd := []string{"exec " + podUID + "_dind" + " docker ps -af name=^" + containerName + "$ --format \"{{.Status}}\""} shell := exec.ExecTask{ Command: "docker", @@ -65,32 +63,30 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { break } + log.G(h.Ctx).Info("\u2705 [STATUS CALL] Status of the container retrieved successfully") + containerstatus := strings.Split(execReturn.Stdout, " ") - // TODO: why first container? if execReturn.Stdout != "" { + log.G(h.Ctx).Info("\u2705 [STATUS CALL] The container " + container.Name + " is in the state: " + containerstatus[0]) + if containerstatus[0] == "Created" { - log.G(h.Ctx).Info("-- Container " + containerName + " is going ready...") resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } else if containerstatus[0] == "Up" { - log.G(h.Ctx).Info("-- Container " + containerName + " is running") resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) } else if containerstatus[0] == "Exited" { - log.G(h.Ctx).Info("-- Container " + containerName + " has been stopped") containerExitCode := strings.Split(containerstatus[1], "(") exitCode, err := strconv.Atoi(strings.Trim(containerExitCode[1], ")")) if err != nil { log.G(h.Ctx).Error(err) exitCode = 0 } - log.G(h.Ctx).Info("-- Container exit code is: " + strconv.Itoa(exitCode)) resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{ExitCode: int32(exitCode)}}, Ready: false}) // release all the GPUs from the container h.GpuManager.Release(containerName) } } else { - log.G(h.Ctx).Info("-- Container " + containerName + " doesn't exist") - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false}) + resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } } } diff --git a/pkg/docker/aux.go b/pkg/docker/aux.go index 1249136..93cc0cf 100644 --- a/pkg/docker/aux.go +++ b/pkg/docker/aux.go @@ -11,8 +11,6 @@ import ( "github.com/containerd/containerd/log" v1 "k8s.io/api/core/v1" - "fmt" - commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/gpustrategies" ) @@ -23,26 +21,16 @@ type SidecarHandler struct { GpuManager gpustrategies.GPUManagerInterface } -func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.InterLinkConfig, data []commonIL.RetrievedPodData, container v1.Container) ([]string, []string, []string, error) { - - podUID := "" - podNamespace := "" +func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.InterLinkConfig, podUID string, podNamespace string, container v1.Container) ([]string, []string, []string, error) { - for _, podData := range data { - podUID = string(podData.Pod.UID) - podNamespace = string(podData.Pod.Namespace) - - // check if the directory exists, if not create it - dirPath := config.DataRootFolder + podData.Pod.Namespace + "-" + podUID - if _, err := os.Stat(dirPath); os.IsNotExist(err) { - err := os.MkdirAll(dirPath, os.ModePerm) - if err != nil { - log.G(Ctx).Error(err) - } else { - log.G(Ctx).Info("-- Created directory " + dirPath) - } + dirPath := config.DataRootFolder + podNamespace + "-" + podUID + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + err := os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + log.G(Ctx).Error(err) + } else { + log.G(Ctx).Info("-- Created directory " + dirPath) } - } if container.Command == nil { @@ -53,7 +41,6 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.Int wd, err := os.Getwd() if err != nil { - log.G(Ctx).Error(err) return nil, nil, nil, err } @@ -63,7 +50,6 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.Int if len(container.Args) == 0 { fileNamePath := filepath.Join(wd, config.DataRootFolder+podNamespace+"-"+podUID, fileName) - log.G(Ctx).Info("Creating file " + fileNamePath) err = os.WriteFile(fileNamePath, []byte(strings.Join(container.Command, " ")), 0644) if err != nil { log.G(Ctx).Error(err) @@ -74,7 +60,6 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.Int argsFileName := container.Name + "_args" argsFileNamePath := filepath.Join(wd, config.DataRootFolder+podNamespace+"-"+podUID, argsFileName) - log.G(Ctx).Info("Creating file " + argsFileNamePath) err = os.WriteFile(argsFileNamePath, []byte(strings.Join(container.Args, " ")), 0644) if err != nil { log.G(Ctx).Error(err) @@ -83,14 +68,12 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.Int fullFileContent := strings.Join(container.Command, " ") + " \"$(cat " + argsFileName + ")\"" fullFileNamePath := filepath.Join(wd, config.DataRootFolder+podNamespace+"-"+podUID, fileName) - log.G(Ctx).Info("Creating file " + fullFileNamePath) err = os.WriteFile(fullFileNamePath, []byte(fullFileContent), 0644) if err != nil { log.G(Ctx).Error(err) return nil, nil, nil, err } - // mount also the args file return []string{"-v " + argsFileNamePath + ":/" + argsFileName, "-v " + fullFileNamePath + ":/" + fileName}, []string{"/bin/sh" + " /" + fileName}, []string{}, nil } else { @@ -98,85 +81,60 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.Int } } -// prepareMounts iterates along the struct provided in the data parameter and checks for ConfigMaps, Secrets and EmptyDirs to be mounted. -// For each element found, the mountData function is called. -// It returns a string composed as the docker -v command to bind mount directories and files and the first encountered error. -func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data []commonIL.RetrievedPodData, container v1.Container) (string, error) { - log.G(Ctx).Info("- Preparing mountpoints for " + container.Name) +func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data commonIL.RetrievedPodData, container v1.Container) (string, error) { mountedData := "" - for _, podData := range data { - - log.G(Ctx).Info("-- pod data ", podData) - - podUID := string(podData.Pod.UID) - podNamespace := string(podData.Pod.UID) - - err := os.MkdirAll(config.DataRootFolder+podData.Pod.Namespace+"-"+podUID, os.ModePerm) - if err != nil { - log.G(Ctx).Error(err) - return "", err - } else { - log.G(Ctx).Info("-- Created directory " + config.DataRootFolder + podData.Pod.Namespace + "-" + podUID) - } - - // print the len of the init containers - log.G(Ctx).Info("Init containers: " + fmt.Sprintf("%+v", podData.InitContainers)) + podUID := string(data.Pod.UID) + podNamespace := string(data.Pod.UID) - allContainers := append(podData.Containers, podData.InitContainers...) - - log.G(Ctx).Info("All containers: " + fmt.Sprintf("%+v", allContainers)) + err := os.MkdirAll(config.DataRootFolder+data.Pod.Namespace+"-"+podUID, os.ModePerm) + if err != nil { + return "", err + } - for _, cont := range allContainers { + allContainers := append(data.Containers, data.InitContainers...) - if cont.Name != container.Name { - continue - } + for _, cont := range allContainers { - containerName := podNamespace + "-" + podUID + "-" + container.Name + if cont.Name != container.Name { + continue + } - log.G(Ctx).Info("cont values: " + fmt.Sprintf("%+v", cont)) + containerName := podNamespace + "-" + podUID + "-" + container.Name - log.G(Ctx).Info("-- Inside Preparing mountpoints for " + cont.Name) - for _, cfgMap := range cont.ConfigMaps { - if containerName == podNamespace+"-"+podUID+"-"+cont.Name { - log.G(Ctx).Info("-- Mounting ConfigMap " + cfgMap.Name) - paths, err := mountData(Ctx, config, podData.Pod, cfgMap, container) - log.G(Ctx).Info("-- Paths: " + strings.Join(paths, ",")) - if err != nil { - log.G(Ctx).Error("Error mounting ConfigMap " + cfgMap.Name) - return "", errors.New("Error mounting ConfigMap " + cfgMap.Name) - } - for _, path := range paths { - mountedData += "-v " + path + " " - } + for _, cfgMap := range cont.ConfigMaps { + if containerName == podNamespace+"-"+podUID+"-"+cont.Name { + paths, err := mountData(Ctx, config, data.Pod, cfgMap, container) + if err != nil { + return "", errors.New("Error mounting ConfigMap " + cfgMap.Name) + } + for _, path := range paths { + mountedData += "-v " + path + " " } } + } - for _, secret := range cont.Secrets { - if containerName == podNamespace+"-"+podUID+"-"+cont.Name { - paths, err := mountData(Ctx, config, podData.Pod, secret, container) - if err != nil { - log.G(Ctx).Error("Error mounting Secret " + secret.Name) - return "", errors.New("Error mounting Secret " + secret.Name) - } - for _, path := range paths { - mountedData += "-v " + path + " " - } + for _, secret := range cont.Secrets { + if containerName == podNamespace+"-"+podUID+"-"+cont.Name { + paths, err := mountData(Ctx, config, data.Pod, secret, container) + if err != nil { + return "", errors.New("Error mounting Secret " + secret.Name) + } + for _, path := range paths { + mountedData += "-v " + path + " " } } + } - for _, emptyDir := range cont.EmptyDirs { - log.G(Ctx).Info("-- EmptyDir to handle " + emptyDir) - if containerName == podNamespace+"-"+podUID+"-"+cont.Name { - paths, err := mountData(Ctx, config, podData.Pod, emptyDir, container) - if err != nil { - log.G(Ctx).Error("Error mounting EmptyDir " + emptyDir) - return "", errors.New("Error mounting EmptyDir " + emptyDir) - } - for _, path := range paths { - mountedData += "-v " + path + " " - } + for _, emptyDir := range cont.EmptyDirs { + if containerName == podNamespace+"-"+podUID+"-"+cont.Name { + paths, err := mountData(Ctx, config, data.Pod, emptyDir, container) + if err != nil { + log.G(Ctx).Error("Error mounting EmptyDir " + emptyDir) + return "", errors.New("Error mounting EmptyDir " + emptyDir) + } + for _, path := range paths { + mountedData += "-v " + path + " " } } } @@ -188,10 +146,6 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data [] return mountedData, nil } -// mountData is called by prepareMounts and creates files and directory according to their definition in the pod structure. -// The data parameter is an interface and it can be of type v1.ConfigMap, v1.Secret and string (for the empty dir). -// Returns a string which is a bind mount of the file/directory. Example: path/to/file/on/host:path/to/file/in/container. -// It also returns the first encountered error. func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, data interface{}, container v1.Container) ([]string, error) { wd, err := os.Getwd() if err != nil { @@ -199,16 +153,8 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, return nil, err } - log.G(Ctx).Info("Inside mountData ") - - //if config.ExportPodData { - - log.G(Ctx).Info("Mounting data for " + container.Name) - for _, mountSpec := range container.VolumeMounts { - log.G(Ctx).Info("Mounting " + mountSpec.Name + " at " + mountSpec.MountPath) - var podVolumeSpec *v1.VolumeSource for _, vol := range pod.Spec.Volumes { @@ -222,9 +168,9 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, err := os.RemoveAll(config.DataRootFolder + pod.Namespace + "-" + string(pod.UID) + "/" + "configMaps/" + vol.Name) if err != nil { - log.G(Ctx).Error("Unable to delete root folder") return nil, err } + if podVolumeSpec != nil && podVolumeSpec.ConfigMap != nil { podConfigMapDir := filepath.Join(wd+"/"+config.DataRootFolder+pod.Namespace+"-"+string(pod.UID)+"/", "configMaps/", vol.Name) mode := os.FileMode(*podVolumeSpec.ConfigMap.DefaultMode) @@ -238,14 +184,8 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, if mount.Data != nil { for key := range mount.Data { - - log.G(Ctx).Info("Key: " + key) - path := filepath.Join(podConfigMapDir, key) path += (":" + correctMountPath + "/" + key + " ") - - log.G(Ctx).Info("Path: " + path) - configMapNamePaths = append(configMapNamePaths, path) } } @@ -259,26 +199,16 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, execReturn, _ := shell.Execute() if execReturn.Stderr != "" { - log.G(Ctx).Error(err) return nil, err - } else { - log.G(Ctx).Debug("-- Created directory " + podConfigMapDir) } - log.G(Ctx).Info("-- Writing ConfigMaps files") for k, v := range mount.Data { // TODO: Ensure that these files are deleted in failure cases fullPath := filepath.Join(podConfigMapDir, k) os.WriteFile(fullPath, []byte(v), mode) if err != nil { - log.G(Ctx).Errorf("Could not write ConfigMap file %s", fullPath) err = os.RemoveAll(fullPath) - if err != nil { - log.G(Ctx).Error("Unable to remove file " + fullPath) - } return nil, err - } else { - log.G(Ctx).Debug("--- Written ConfigMap file " + fullPath) } } return configMapNamePaths, nil @@ -289,7 +219,6 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, err := os.RemoveAll(config.DataRootFolder + pod.Namespace + "-" + string(pod.UID) + "/" + "secrets/" + vol.Name) if err != nil { - log.G(Ctx).Error("Unable to delete root folder") return nil, err } if podVolumeSpec != nil && podVolumeSpec.Secret != nil { @@ -313,30 +242,19 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, execReturn, _ := shell.Execute() if strings.Compare(execReturn.Stdout, "") != 0 { - log.G(Ctx).Error(err) return nil, err } if execReturn.Stderr != "" { - log.G(Ctx).Error(execReturn.Stderr) return nil, errors.New(execReturn.Stderr) - } else { - log.G(Ctx).Debug("-- Created directory " + podSecretDir) } - log.G(Ctx).Info("-- Writing Secret files") for k, v := range mount.Data { // TODO: Ensure that these files are deleted in failure cases fullPath := filepath.Join(podSecretDir, k) os.WriteFile(fullPath, v, mode) if err != nil { - log.G(Ctx).Errorf("Could not write Secret file %s", fullPath) err = os.RemoveAll(fullPath) - if err != nil { - log.G(Ctx).Error("Unable to remove file " + fullPath) - } return nil, err - } else { - log.G(Ctx).Debug("--- Written Secret file " + fullPath) } } return secretNamePaths, nil @@ -349,7 +267,6 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, parts := strings.Split(data.(string), "/") emptyDirName := parts[len(parts)-1] if emptyDirName != vol.Name { - log.G(Ctx).Info("Skipping " + vol.Name + " as it is not the same as " + emptyDirName) continue } @@ -366,12 +283,12 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, if mountSpec.MountPropagation != nil && *mountSpec.MountPropagation == v1.MountPropagationBidirectional { isBidirectional = true } + break } } edPath = filepath.Join(wd + "/" + config.DataRootFolder + pod.Namespace + "-" + string(pod.UID) + "/" + "emptyDirs/" + vol.Name) - log.G(Ctx).Info("-- Creating EmptyDir in " + edPath) cmd := []string{"-p " + edPath} shell := exec2.ExecTask{ Command: "mkdir", @@ -381,13 +298,9 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, _, err := shell.Execute() if err != nil { - log.G(Ctx).Error(err) return []string{""}, nil - } else { - log.G(Ctx).Debug("-- Created EmptyDir in " + edPath) } - // if the emptyDir is read only, append :ro to the path if isReadOnly { edPath += (":" + emptyDirMountPath + "/:ro") } else { @@ -403,6 +316,6 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, } } } - //} + return nil, err } diff --git a/pkg/docker/gpustrategies/NvidiaHandler.go b/pkg/docker/gpustrategies/NvidiaHandler.go index 5c454f0..4b93226 100644 --- a/pkg/docker/gpustrategies/NvidiaHandler.go +++ b/pkg/docker/gpustrategies/NvidiaHandler.go @@ -58,8 +58,6 @@ func (a *GPUManager) Init() error { // Discover implements the Discover function of the GPUManager interface func (a *GPUManager) Discover() error { - log.G(a.Ctx).Info("Discovering GPUs...") - count, ret := nvml.DeviceGetCount() if ret != nvml.SUCCESS { return fmt.Errorf("Unable to get device count: %v", nvml.ErrorString(ret)) @@ -92,12 +90,12 @@ func (a *GPUManager) Discover() error { // print the GPUSpecsList if the length is greater than 0 if len(a.GPUSpecsList) > 0 { - log.G(a.Ctx).Info("Discovered GPUs:") + log.G(a.Ctx).Info("\u2705 Discovered GPUs:") for _, gpuSpec := range a.GPUSpecsList { - log.G(a.Ctx).Info(fmt.Sprintf("Name: %s, UUID: %s, Type: %s, Available: %t, Index: %d", gpuSpec.Name, gpuSpec.UUID, gpuSpec.Type, gpuSpec.Available, gpuSpec.Index)) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Name: %s, UUID: %s, Type: %s, Available: %t, Index: %d", gpuSpec.Name, gpuSpec.UUID, gpuSpec.Type, gpuSpec.Available, gpuSpec.Index)) } } else { - log.G(a.Ctx).Info("No GPUs discovered") + log.G(a.Ctx).Info(" \u2705 No GPUs discovered") } return nil @@ -105,8 +103,6 @@ func (a *GPUManager) Discover() error { func (a *GPUManager) Check() error { - log.G(a.Ctx).Info("Checking the availability of GPUs...") - cli, err := client.NewEnvClient() if err != nil { return fmt.Errorf("unable to create a new Docker client: %v", err) @@ -148,9 +144,9 @@ func (a *GPUManager) Check() error { // print the GPUSpecsList that are not available for _, gpuSpec := range a.GPUSpecsList { if !gpuSpec.Available { - log.G(a.Ctx).Info(fmt.Sprintf("GPU with UUID %s is not available. It is in use by container %s", gpuSpec.UUID, gpuSpec.ContainerID)) + log.G(a.Ctx).Info(fmt.Sprintf("\u274C GPU with UUID %s is not available. It is in use by container %s", gpuSpec.UUID, gpuSpec.ContainerID)) } else { - log.G(a.Ctx).Info(fmt.Sprintf("GPU with UUID %s is available", gpuSpec.UUID)) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 GPU with UUID %s is available", gpuSpec.UUID)) } } @@ -159,8 +155,6 @@ func (a *GPUManager) Check() error { func (a *GPUManager) Shutdown() error { - log.G(a.Ctx).Info("Shutting down NVML...") - ret := nvml.Shutdown() if ret != nvml.SUCCESS { return fmt.Errorf("Unable to shutdown NVML: %v", nvml.ErrorString(ret)) @@ -193,8 +187,6 @@ func (a *GPUManager) Assign(UUID string, containerID string) error { func (a *GPUManager) Release(containerID string) error { - log.G(a.Ctx).Info("Releasing GPU from container " + containerID) - a.GPUSpecsMutex.Lock() defer a.GPUSpecsMutex.Unlock() @@ -210,8 +202,6 @@ func (a *GPUManager) Release(containerID string) error { } } - log.G(a.Ctx).Info("Correctly released GPU from container " + containerID) - return nil } @@ -252,8 +242,6 @@ func (a *GPUManager) GetAndAssignAvailableGPUs(numGPUs int, containerID string) // dump the GPUSpecsList into a JSON file func (a *GPUManager) Dump() error { - log.G(a.Ctx).Info("Dumping the GPUSpecsList into a JSON file...") - // Convert the array to JSON format jsonData, err := json.MarshalIndent(a.GPUSpecsList, "", " ") if err != nil {