Docker prototype (#27)

* Add execution container

* Remove deprecated function call

* Add support for slice script

* Add support for Git authentication

* Outsource image tags

* Add first MVP of Docker executor

* Allow running on a remote Docker host

* Add debug mode to main.go and remove debug mode
from config.go

HadesScheduler/docker/container.go

@@ -1,14 +1,19 @@
 package docker
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
+	"io"
+	"os"
+
 	"github.com/Mtze/HadesCI/hadesScheduler/config"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
-	"os"
+	log "github.com/sirupsen/logrus"
 )
 
 var defaultHostConfig = container.HostConfig{
@@ -42,3 +47,44 @@ func writeContainerLogsToFile(ctx context.Context, client *client.Client, contai
 	_, err = stdcopy.StdCopy(out, out, logReader)
 	return err
 }
+
+func copyFileToContainer(ctx context.Context, client *client.Client, containerID, srcPath, dstPath string) error {
+	scriptFile, err := os.Open(srcPath)
+	if err != nil {
+		return err
+	}
+	defer scriptFile.Close()
+
+	// Create a buffer to hold the tar archive
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	// Read the script content
+	scriptContent, err := io.ReadAll(scriptFile)
+	if err != nil {
+		return err
+	}
+	defer tw.Close()
+
+	// Add the script content to the tar archive
+	tarHeader := &tar.Header{
+		Name: "script.sh",
+		Size: int64(len(scriptContent)),
+		Mode: 0755, // Make sure the script is executable
+	}
+	if err := tw.WriteHeader(tarHeader); err != nil {
+		return err
+	}
+	if _, err := tw.Write(scriptContent); err != nil {
+		return err
+	}
+
+	// Now send the tar archive to CopyToContainer
+	err = client.CopyToContainer(ctx, containerID, dstPath, &buf, types.CopyToContainerOptions{
+		AllowOverwriteDirWithFile: false,
+	})
+	if err != nil {
+		log.WithError(err).Error("Failed to copy script to container")
+		return err
+	}
+	return nil
+}

HadesScheduler/docker/docker.go

@@ -3,11 +3,14 @@ package docker
 import (
 	"context"
 	"fmt"
-	"github.com/Mtze/HadesCI/hadesScheduler/config"
 	"io"
 	"sync"
 	"time"
 
+	"github.com/Mtze/HadesCI/hadesScheduler/config"
+
+	"os"
+
 	"github.com/Mtze/HadesCI/shared/payload"
 	"github.com/Mtze/HadesCI/shared/utils"
 	"github.com/docker/docker/api/types"
@@ -16,17 +19,23 @@ import (
 	"github.com/docker/docker/client"
 	_ "github.com/docker/docker/client"
 	log "github.com/sirupsen/logrus"
-	"os"
 )
 
 var cli *client.Client
 
 type Scheduler struct{}
 
+type DockerConfig struct {
+	DockerHost string `env:"DOCKER_HOST" envDefault:"unix:///var/run/docker.sock"`
+}
+
 func init() {
+	var DockerCfg DockerConfig
+	utils.LoadConfig(&DockerCfg)
+
 	var err error
 	// Create a new Docker client
-	cli, err = client.NewClientWithOpts(client.FromEnv)
+	cli, err = client.NewClientWithOpts(client.WithHost(DockerCfg.DockerHost), client.WithAPIVersionNegotiation())
 	if err != nil {
 		log.WithError(err).Fatal("Failed to create Docker client")
 	}
@@ -52,6 +61,16 @@ func (d Scheduler) ScheduleJob(job payload.BuildJob) error {
 		return err
 	}
 	log.Debugf("Create Shared Volume in %s", time.Since(startOfVolume))
+	// Delete the shared volume after the job is done
+	defer func() {
+		time.Sleep(1 * time.Second)
+		startOfDelete := time.Now()
+		err = deleteSharedVolume(ctx, cli, config.SharedVolumeName)
+		if err != nil {
+			log.WithError(err).Error("Failed to delete shared volume")
+		}
+		log.Debugf("Delete Shared Volume in %s", time.Since(startOfDelete))
+	}()
 
 	startOfClone := time.Now()
 	// Clone the repository
@@ -194,6 +213,13 @@ func executeRepository(ctx context.Context, client *client.Client, buildConfig p
 		return err
 	}
 
+	// Copy the script to the container
+	err = copyFileToContainer(ctx, client, resp.ID, scriptPath, "/tmp")
+	if err != nil {
+		log.WithError(err).Error("Failed to copy script to container")
+		return err
+	}
+
 	// Start the container
 	err = client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
 	if err != nil {

HadesScheduler/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/Mtze/HadesCI/hadesScheduler/docker"
 	"github.com/Mtze/HadesCI/hadesScheduler/kube"
@@ -19,6 +20,11 @@ type JobScheduler interface {
 }
 
 func main() {
+	if is_debug := os.Getenv("DEBUG"); is_debug == "true" {
+		log.SetLevel(log.DebugLevel)
+		log.Warn("DEBUG MODE ENABLED")
+	}
+
 	var cfg utils.RabbitMQConfig
 	utils.LoadConfig(&cfg)
 

shared/go.mod

@@ -3,14 +3,13 @@ module github.com/Mtze/HadesCI/shared
 go 1.21.0
 
 require (
+	github.com/caarlos0/env/v9 v9.0.0
+	github.com/joho/godotenv v1.5.1
 	github.com/rabbitmq/amqp091-go v1.8.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.4
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

shared/go.sum

@@ -1,10 +1,12 @@
+github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc=
+github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -24,7 +26,6 @@ go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

shared/utils/config.go

@@ -1,8 +1,6 @@
 package utils
 
 import (
-	"os"
-
 	"github.com/caarlos0/env/v9"
 	"github.com/joho/godotenv"
 	log "github.com/sirupsen/logrus"
@@ -23,12 +21,6 @@ type ExecutorConfig struct {
 }
 
 func LoadConfig(cfg interface{}) {
-
-	if is_debug := os.Getenv("DEBUG"); is_debug == "true" {
-		log.SetLevel(log.DebugLevel)
-		log.Warn("DEBUG MODE ENABLED")
-	}
-
 	err := godotenv.Load()
 	if err != nil {
 		log.WithError(err).Warn("Error loading .env file")