From 3189bd9ec343d1c96eb2a0507f4de096e825eb95 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 29 Jul 2024 17:59:30 +0100 Subject: [PATCH 01/14] temp commit --- backend/middleware/basic.go | 1 + backend/middleware/jwt.go | 1 + backend/migrations/20240729155442.sql | 13 +++++ backend/migrations/20240729155926.sql | 2 + backend/migrations/20240729160028.sql | 2 + backend/migrations/atlas.sum | 5 +- backend/models/artefact.go | 15 ++++++ backend/models/storage.go | 11 ++++ ee/backend/controllers/artefacts.go | 76 +++++++++++++++++++++++++++ ee/backend/main.go | 3 ++ 10 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 backend/migrations/20240729155442.sql create mode 100644 backend/migrations/20240729155926.sql create mode 100644 backend/migrations/20240729160028.sql create mode 100644 backend/models/artefact.go create mode 100644 ee/backend/controllers/artefacts.go diff --git a/backend/middleware/basic.go b/backend/middleware/basic.go index 5313110a8..321bc792c 100644 --- a/backend/middleware/basic.go +++ b/backend/middleware/basic.go @@ -59,6 +59,7 @@ func HttpBasicApiAuth() gin.HandlerFunc { } else { setDefaultOrganisationId(c) c.Set(ACCESS_LEVEL_KEY, jobToken.Type) + c.Set(JOB_TOKEN_KEY, jobToken.Value) } } else if token == os.Getenv("BEARER_AUTH_TOKEN") { setDefaultOrganisationId(c) diff --git a/backend/middleware/jwt.go b/backend/middleware/jwt.go index 297703eab..70cfee2fa 100644 --- a/backend/middleware/jwt.go +++ b/backend/middleware/jwt.go @@ -309,3 +309,4 @@ func CORSMiddleware() gin.HandlerFunc { const ORGANISATION_ID_KEY = "organisation_ID" const ACCESS_LEVEL_KEY = "access_level" +const JOB_TOKEN_KEY = "job_token" diff --git a/backend/migrations/20240729155442.sql b/backend/migrations/20240729155442.sql new file mode 100644 index 000000000..b382326de --- /dev/null +++ b/backend/migrations/20240729155442.sql @@ -0,0 +1,13 @@ +-- Create "job_artefacts" table +CREATE TABLE "public"."job_artefacts" ( + "id" bigserial NOT NULL, + "created_at" timestamptz NULL, + "updated_at" timestamptz NULL, + "deleted_at" timestamptz NULL, + "job_token_id" bigint NULL, + "contents" bytea NULL, + PRIMARY KEY ("id"), + CONSTRAINT "fk_job_artefacts_job_token" FOREIGN KEY ("job_token_id") REFERENCES "public"."job_tokens" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION +); +-- Create index "idx_job_artefacts_deleted_at" to table: "job_artefacts" +CREATE INDEX "idx_job_artefacts_deleted_at" ON "public"."job_artefacts" ("deleted_at"); diff --git a/backend/migrations/20240729155926.sql b/backend/migrations/20240729155926.sql new file mode 100644 index 000000000..7544af0ac --- /dev/null +++ b/backend/migrations/20240729155926.sql @@ -0,0 +1,2 @@ +-- Modify "job_artefacts" table +ALTER TABLE "public"."job_artefacts" ADD COLUMN "size" bigint NULL, ADD COLUMN "content_type" text NULL; diff --git a/backend/migrations/20240729160028.sql b/backend/migrations/20240729160028.sql new file mode 100644 index 000000000..060e1cc3b --- /dev/null +++ b/backend/migrations/20240729160028.sql @@ -0,0 +1,2 @@ +-- Modify "job_artefacts" table +ALTER TABLE "public"."job_artefacts" ADD COLUMN "filename" text NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index fb989e498..3c88acc75 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:r0O5Hy6gJdMDgkCW9ocd4ytiCyPghNcXKTAT3tuppZ8= +h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -27,3 +27,6 @@ h1:r0O5Hy6gJdMDgkCW9ocd4ytiCyPghNcXKTAT3tuppZ8= 20240704192835.sql h1:F84HKuE3qX8hYqDCU2K9NDG4WQaSfmnk2P4OCje54qg= 20240705144450.sql h1:AAB4GbML2Jea1MSv/+E0DLMMCI9GLqwptpvBScvU19s= 20240709165155.sql h1:HWDuhwXD+esr9uaKHsqwltPfUEq5uLdcEzpnZ1biKgU= +20240729155442.sql h1:s7PCALP3SgPz5y9Ya7HkDzYjeN86Q5NniW2YIkOMBrQ= +20240729155926.sql h1:8vsDrpy/R1UDI+meIp6KoDfhS60t+ngu8aPB+uonFZ4= +20240729160028.sql h1:snkkxhA2aEQhqBmIhN8l+nPlBhrPOZiPP+dnyhobwD8= diff --git a/backend/models/artefact.go b/backend/models/artefact.go new file mode 100644 index 000000000..46855b781 --- /dev/null +++ b/backend/models/artefact.go @@ -0,0 +1,15 @@ +package models + +import ( + "gorm.io/gorm" +) + +type JobArtefact struct { + gorm.Model + JobTokenID uint + JobToken JobToken + Filename string + Contents []byte `gorm:"type:bytea"` + Size int64 + ContentType string +} diff --git a/backend/models/storage.go b/backend/models/storage.go index 599718ad3..890e92f00 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -1126,6 +1126,17 @@ func (db *Database) GetJobToken(tenantId any) (*JobToken, error) { return token, nil } +func (db *Database) DeleteJobTokenArtefacts(jobTokenId uint) error { + artefact := JobArtefact{} + result := db.GormDB.Where("job_token_id = ?", jobTokenId).Delete(&artefact) + if result.Error != nil { + return result.Error + } + log.Printf("DeleteJobTokenArtefacts %v has been deleted successfully\n", jobTokenId) + return nil + +} + func (db *Database) CreateGithubAppInstallation(installationId int64, githubAppId int64, login string, accountId int, repoFullName string) (*GithubAppInstallation, error) { installation := &GithubAppInstallation{ GithubInstallationId: installationId, diff --git a/ee/backend/controllers/artefacts.go b/ee/backend/controllers/artefacts.go new file mode 100644 index 000000000..2dcb589d1 --- /dev/null +++ b/ee/backend/controllers/artefacts.go @@ -0,0 +1,76 @@ +package controllers + +import ( + "github.com/diggerhq/digger/backend/middleware" + "github.com/diggerhq/digger/backend/models" + "github.com/gin-gonic/gin" + "io" + "log" + "net/http" +) + +func SetJobArtefact(c *gin.Context) { + jobTokenValue, exists := c.Get(middleware.JOB_TOKEN_KEY) + if !exists { + c.String(http.StatusBadRequest, "missing value: bearer job token") + return + } + + jobToken, err := models.DB.GetJobToken(jobTokenValue) + if err != nil { + c.String(http.StatusBadRequest, "could not find job token") + return + } + + file, err := c.FormFile("file") + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "No file is received"}) + return + } + + // Open the file + src, err := file.Open() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error opening file"}) + return + } + defer src.Close() + + // Read the content + content, err := io.ReadAll(src) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error reading file content"}) + return + } + + // Deleting existing artefacts + err = models.DB.DeleteJobTokenArtefacts(jobToken.ID) + if err != nil { + log.Printf("could not delete artefacts: %v", err) + c.JSON(http.StatusInternalServerError, "could not delete existing artefacts") + return + } + + log.Printf("contents of the file is: %v", string(content)) + // Create a new File record + artefactRecord := models.JobArtefact{ + JobTokenID: jobToken.ID, + Filename: file.Filename, + Contents: content, + Size: file.Size, + ContentType: file.Header.Get("Content-Type"), + } + + // Save the file to the database + if result := models.DB.GormDB.Create(&artefactRecord); result.Error != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error saving file to database"}) + return + } + + c.JSON(http.StatusOK, gin.H{"message": "File uploaded successfully", "id": artefactRecord.ID}) + +} + +func DownloadJobArtefact(c *gin.Context) { + +} diff --git a/ee/backend/main.go b/ee/backend/main.go index f1bc66160..15db95fc0 100644 --- a/ee/backend/main.go +++ b/ee/backend/main.go @@ -79,6 +79,9 @@ func main() { policiesGroup.GET("/:policyid/details", web.PolicyDetailsPage) policiesGroup.POST("/:policyid/details", web.PolicyDetailsUpdatePage) + jobArtefactsGroup := r.Group("/job_artefacts") + jobArtefactsGroup.Use(middleware.GetApiMiddleware()) + jobArtefactsGroup.PUT("/", controllers.SetJobArtefact) port := config.GetPort() r.Run(fmt.Sprintf(":%d", port)) } From dafbca09557023a603fd4e99af6e572b0c6f1618 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 29 Jul 2024 18:14:29 +0100 Subject: [PATCH 02/14] download from artefact store --- backend/models/storage.go | 8 ++++++++ ee/backend/controllers/artefacts.go | 28 +++++++++++++++++++++++++++- ee/backend/main.go | 2 ++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/backend/models/storage.go b/backend/models/storage.go index 890e92f00..c3b5622ac 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -1137,6 +1137,14 @@ func (db *Database) DeleteJobTokenArtefacts(jobTokenId uint) error { } +func (db *Database) GetJobArtefact(jobTokenId uint) (*JobArtefact, error) { + var artefact JobArtefact + if err := DB.GormDB.Where("job_token_id = ?", jobTokenId).First(&artefact).Error; err != nil { + return nil, err + } + return &artefact, nil +} + func (db *Database) CreateGithubAppInstallation(installationId int64, githubAppId int64, login string, accountId int, repoFullName string) (*GithubAppInstallation, error) { installation := &GithubAppInstallation{ GithubInstallationId: installationId, diff --git a/ee/backend/controllers/artefacts.go b/ee/backend/controllers/artefacts.go index 2dcb589d1..94dbb3097 100644 --- a/ee/backend/controllers/artefacts.go +++ b/ee/backend/controllers/artefacts.go @@ -1,9 +1,11 @@ package controllers import ( + "fmt" "github.com/diggerhq/digger/backend/middleware" "github.com/diggerhq/digger/backend/models" "github.com/gin-gonic/gin" + "github.com/google/uuid" "io" "log" "net/http" @@ -55,7 +57,7 @@ func SetJobArtefact(c *gin.Context) { // Create a new File record artefactRecord := models.JobArtefact{ JobTokenID: jobToken.ID, - Filename: file.Filename, + Filename: uuid.NewString(), Contents: content, Size: file.Size, ContentType: file.Header.Get("Content-Type"), @@ -72,5 +74,29 @@ func SetJobArtefact(c *gin.Context) { } func DownloadJobArtefact(c *gin.Context) { + jobTokenValue, exists := c.Get(middleware.JOB_TOKEN_KEY) + if !exists { + c.String(http.StatusBadRequest, "missing value: bearer job token") + return + } + + jobToken, err := models.DB.GetJobToken(jobTokenValue) + if err != nil { + c.String(http.StatusBadRequest, "could not find job token") + return + } + + artefact, err := models.DB.GetJobArtefact(jobToken.ID) + if err != nil { + c.String(http.StatusBadRequest, "could not find any artefacts for job token") + return + } + + // Set the appropriate headers + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", artefact.Filename)) + c.Header("Content-Type", artefact.ContentType) + c.Header("Content-Length", fmt.Sprintf("%d", artefact.Size)) + // Write the file contents to the response + c.Data(http.StatusOK, artefact.ContentType, artefact.Contents) } diff --git a/ee/backend/main.go b/ee/backend/main.go index 15db95fc0..1cb99cfc9 100644 --- a/ee/backend/main.go +++ b/ee/backend/main.go @@ -82,6 +82,8 @@ func main() { jobArtefactsGroup := r.Group("/job_artefacts") jobArtefactsGroup.Use(middleware.GetApiMiddleware()) jobArtefactsGroup.PUT("/", controllers.SetJobArtefact) + jobArtefactsGroup.GET("/", controllers.DownloadJobArtefact) + port := config.GetPort() r.Run(fmt.Sprintf(":%d", port)) } From 916d8cfdaeca47c39f3a65bd689893af8dbcf3e0 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 11:04:27 +0100 Subject: [PATCH 03/14] zip and upload artefact --- dgctl/cmd/exec.go | 18 +++++++ dgctl/utils/backendutils.go | 77 +++++++++++++++++++++++++++++ dgctl/utils/gitio.go | 99 +++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 dgctl/utils/backendutils.go create mode 100644 dgctl/utils/gitio.go diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index 0c4dcae26..cfd0483b3 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/diggerhq/digger/dgctl/utils" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/libs/spec" "github.com/google/go-github/v61/github" @@ -273,6 +274,23 @@ var execCmd = &cobra.Command{ var spec spec.Spec err = json.Unmarshal(specBytes, &spec) + // attatch zip archive to backend + backendToken := spec.Job.BackendJobToken + zipLocation, err := utils.ArchiveGitRepo("./") + statusCode, respBody, err := utils.SendZipAsJobArtefact(diggerHostname, zipLocation, backendToken) + if err != nil { + log.Printf("could not attach zip artefact: %v", err) + os.Exit(1) + } + if *statusCode != 200 { + log.Printf("unexpected status code from backend: %v", *statusCode) + log.Printf("server response: %v", *respBody) + os.Exit(1) + } + + log.Printf("exiting early ...") + os.Exit(0) + token := os.Getenv("GITHUB_PAT_TOKEN") if token == "" { log.Printf("missing variable: GITHUB_PAT_TOKEN") diff --git a/dgctl/utils/backendutils.go b/dgctl/utils/backendutils.go new file mode 100644 index 000000000..684923abf --- /dev/null +++ b/dgctl/utils/backendutils.go @@ -0,0 +1,77 @@ +package utils + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" +) + +func SendZipAsJobArtefact(backendUrl string, zipLocation string, jobToken string) (*int, *string, error) { + url := fmt.Sprintf("%v/job_artefacts/", backendUrl) + filePath := zipLocation + + // Open the file + file, err := os.Open(filePath) + if err != nil { + fmt.Println("Error opening file:", err) + return nil, nil, fmt.Errorf("Error opening file:", err) + } + defer file.Close() + + // Create a buffer to store our request body as bytes + var requestBody bytes.Buffer + + // Create a multipart writer + multipartWriter := multipart.NewWriter(&requestBody) + + // Create a form file writer for our file field + fileWriter, err := multipartWriter.CreateFormFile("file", filepath.Base(filePath)) + if err != nil { + fmt.Println("Error creating form file:", err) + return nil, nil, fmt.Errorf("Error creating form file:", err) + } + + // Copy the file content to the form file writer + _, err = io.Copy(fileWriter, file) + if err != nil { + fmt.Println("Error copying file content:", err) + return nil, nil, fmt.Errorf("Error copying file content:", err) + } + + // Close the multipart writer to finalize the request body + multipartWriter.Close() + + // Create a new HTTP request + req, err := http.NewRequest("PUT", url, &requestBody) + if err != nil { + fmt.Println("Error creating request:", err) + return nil, nil, fmt.Errorf("Error creating request:", err) + } + + // Set the content type header + req.Header.Set("Content-Type", multipartWriter.FormDataContentType()) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", jobToken)) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + fmt.Println("Error sending request:", err) + return nil, nil, fmt.Errorf("Error sending request:", err) + } + defer resp.Body.Close() + + // Read and print the response + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response:", err) + return nil, nil, fmt.Errorf("Error reading response: %v", err) + } + + b := string(body) + return &resp.StatusCode, &b, nil +} diff --git a/dgctl/utils/gitio.go b/dgctl/utils/gitio.go new file mode 100644 index 000000000..97eeec4c4 --- /dev/null +++ b/dgctl/utils/gitio.go @@ -0,0 +1,99 @@ +package utils + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "time" +) + +func ArchiveGitRepo(sourcePath string) (string, error) { + // Generate a unique ID for the temp directory + tempID := fmt.Sprintf("%d", time.Now().UnixNano()) + tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("temp_%s", tempID)) + + // Create the temp directory + if err := os.MkdirAll(tempDir, 0755); err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + + // Copy the source directory to the temp location + if err := copyDir(sourcePath, tempDir); err != nil { + return "", fmt.Errorf("failed to copy directory: %w", err) + } + + // Initialize a new git repo in the copied location + cmd := exec.Command("git", "init") + cmd.Dir = tempDir + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to initialize git repo: %w", err) + } + + // Add all files to the git repo + cmd = exec.Command("git", "add", ".") + cmd.Dir = tempDir + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to add files to git repo: %w", err) + } + + // Commit the files + cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd.Dir = tempDir + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to commit files: %w", err) + } + + // Create a zip file using git archive + zipFile := filepath.Join(os.TempDir(), fmt.Sprintf("archive_%s.zip", tempID)) + cmd = exec.Command("git", "archive", "--format=zip", "-o", zipFile, "HEAD") + cmd.Dir = tempDir + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to create zip archive: %w", err) + } + + return zipFile, nil +} + +func copyDir(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Skip .git directory + if info.IsDir() && info.Name() == ".git" { + return filepath.SkipDir + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + dstPath := filepath.Join(dst, relPath) + + if info.IsDir() { + return os.MkdirAll(dstPath, info.Mode()) + } + + return copyFile(path, dstPath) + }) +} + +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + return err +} From 90d9b8ee17bbe35a563b93ebdbd46ec3c7a87684 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 11:06:08 +0100 Subject: [PATCH 04/14] fix url path --- dgctl/utils/backendutils.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dgctl/utils/backendutils.go b/dgctl/utils/backendutils.go index 684923abf..99182b73b 100644 --- a/dgctl/utils/backendutils.go +++ b/dgctl/utils/backendutils.go @@ -6,12 +6,19 @@ import ( "io" "mime/multipart" "net/http" + "net/url" "os" + "path" "path/filepath" ) func SendZipAsJobArtefact(backendUrl string, zipLocation string, jobToken string) (*int, *string, error) { - url := fmt.Sprintf("%v/job_artefacts/", backendUrl) + u, err := url.Parse(backendUrl) + if err != nil { + return nil, nil, err + } + u.Path = path.Join(u.Path, "job_artefacts") + url := u.String() filePath := zipLocation // Open the file From d77cbc8866458ab8c64024b978f910ca6f60279c Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 11:24:47 +0100 Subject: [PATCH 05/14] refactor to lib --- dgctl/cmd/exec.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index cfd0483b3..e1888c2c0 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "github.com/diggerhq/digger/dgctl/utils" + "github.com/diggerhq/digger/libs/backendapi" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/libs/spec" "github.com/google/go-github/v61/github" @@ -274,10 +275,11 @@ var execCmd = &cobra.Command{ var spec spec.Spec err = json.Unmarshal(specBytes, &spec) - // attatch zip archive to backend + // attach zip archive to backend backendToken := spec.Job.BackendJobToken zipLocation, err := utils.ArchiveGitRepo("./") - statusCode, respBody, err := utils.SendZipAsJobArtefact(diggerHostname, zipLocation, backendToken) + statusCode, respBody, err := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken}.UploadJobArtefact(zipLocation) + if err != nil { log.Printf("could not attach zip artefact: %v", err) os.Exit(1) @@ -288,9 +290,6 @@ var execCmd = &cobra.Command{ os.Exit(1) } - log.Printf("exiting early ...") - os.Exit(0) - token := os.Getenv("GITHUB_PAT_TOKEN") if token == "" { log.Printf("missing variable: GITHUB_PAT_TOKEN") @@ -307,11 +306,6 @@ var execCmd = &cobra.Command{ os.Exit(1) } } - err = pushToBranch(prBranch) - if err != nil { - log.Printf("could not push to branchL %v", err) - os.Exit(1) - } inputs := orchestrator_scheduler.WorkflowInput{ Spec: string(specBytes), From 1c833980c85ff3f05edbf59bc1ac09c4f035adc9 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 12:10:01 +0100 Subject: [PATCH 06/14] cli: download and chdir to artefact --- cli/pkg/spec/spec.go | 26 +++++- cli/pkg/utils/io.go | 52 +++++++++++ dgctl/cmd/exec.go | 6 +- ee/backend/controllers/artefacts.go | 2 +- libs/backendapi/backend.go | 2 + libs/backendapi/diggerapi.go | 140 ++++++++++++++++++++++++++++ libs/backendapi/mocks.go | 8 ++ 7 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 cli/pkg/utils/io.go diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 528ebcb55..bde6057cf 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/diggerhq/digger/cli/pkg/digger" "github.com/diggerhq/digger/cli/pkg/usage" + "github.com/diggerhq/digger/cli/pkg/utils" "github.com/diggerhq/digger/libs/ci" "github.com/diggerhq/digger/libs/comment_utils/reporting" comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary" @@ -14,6 +15,8 @@ import ( "github.com/samber/lo" "log" "os" + "os/exec" + "strings" "time" ) @@ -169,6 +172,27 @@ func RunSpecManualCommand( usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1) } + // download zip artefact, git init and prepare for job execution + tempDir, err := os.MkdirTemp("", "downloaded-zip-") + if err != nil { + log.Printf("failed to create temp dir: %w", err) + os.Exit(1) + } + + // downloading artefact zip , extracting, git init and then chdir to that directory for job execution + absoluteFileName, err := backendApi.DownloadJobArtefact(tempDir) + if err != nil { + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not download job artefact: %v", err), 1) + } + zipPath := *absoluteFileName + utils.ExtractZip(zipPath, tempDir) + // Transforming: /var/temp/xxx/yyy/blabla.zip -> /var/temp/xxx/yyy/blabla + gitLocation := strings.TrimSuffix(zipPath, ".zip") + os.Chdir(gitLocation) + cmd := exec.Command("git", "init") + cmd.Stdout = os.Stdout + cmd.Stdout = os.Stderr + policyChecker, err := policyProvider.GetPolicyProvider(spec.Policy, spec.Backend.BackendHostname, spec.Backend.BackendOrganisationName, spec.Backend.BackendJobToken) if err != nil { usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get policy provider: %v", err), 1) @@ -199,7 +223,7 @@ func RunSpecManualCommand( } commentUpdater := comment_summary.NoopCommentUpdater{} - // TODO: do not require conversion to gh service + // do not change these placeholders as they are parsed by dgctl to stream logs log.Printf("<========= DIGGER RUNNING IN MANUAL MODE =========>") allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, false, false, commentId, currentDir) log.Printf("<========= DIGGER COMPLETED =========>") diff --git a/cli/pkg/utils/io.go b/cli/pkg/utils/io.go new file mode 100644 index 000000000..70d764f2a --- /dev/null +++ b/cli/pkg/utils/io.go @@ -0,0 +1,52 @@ +package utils + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path/filepath" +) + +func ExtractZip(zipFilePath string, outDir string) error { + + // Open the zip file + zipReader, err := zip.OpenReader(zipFilePath) + if err != nil { + return fmt.Errorf("failed to open zip: %w", err) + } + defer zipReader.Close() + + for _, file := range zipReader.File { + path := filepath.Join(outDir, file.Name) + + if file.FileInfo().IsDir() { + os.MkdirAll(path, os.ModePerm) + continue + } + + if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + dstFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + + srcFile, err := file.Open() + if err != nil { + dstFile.Close() + return fmt.Errorf("failed to open zip file: %w", err) + } + + _, err = io.Copy(dstFile, srcFile) + srcFile.Close() + dstFile.Close() + + if err != nil { + return fmt.Errorf("failed to extract file: %w", err) + } + } + return nil +} diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index e1888c2c0..37ed886fb 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -278,8 +278,8 @@ var execCmd = &cobra.Command{ // attach zip archive to backend backendToken := spec.Job.BackendJobToken zipLocation, err := utils.ArchiveGitRepo("./") - statusCode, respBody, err := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken}.UploadJobArtefact(zipLocation) - + backendApi := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken} + statusCode, respBody, err := backendApi.UploadJobArtefact(zipLocation) if err != nil { log.Printf("could not attach zip artefact: %v", err) os.Exit(1) @@ -289,7 +289,7 @@ var execCmd = &cobra.Command{ log.Printf("server response: %v", *respBody) os.Exit(1) } - + token := os.Getenv("GITHUB_PAT_TOKEN") if token == "" { log.Printf("missing variable: GITHUB_PAT_TOKEN") diff --git a/ee/backend/controllers/artefacts.go b/ee/backend/controllers/artefacts.go index 94dbb3097..f4d359d9a 100644 --- a/ee/backend/controllers/artefacts.go +++ b/ee/backend/controllers/artefacts.go @@ -57,7 +57,7 @@ func SetJobArtefact(c *gin.Context) { // Create a new File record artefactRecord := models.JobArtefact{ JobTokenID: jobToken.ID, - Filename: uuid.NewString(), + Filename: uuid.NewString() + ".zip", Contents: content, Size: file.Size, ContentType: file.Header.Get("Content-Type"), diff --git a/libs/backendapi/backend.go b/libs/backendapi/backend.go index ebbc5eedf..f97902c5c 100644 --- a/libs/backendapi/backend.go +++ b/libs/backendapi/backend.go @@ -10,4 +10,6 @@ type Api interface { ReportProject(repo string, projectName string, configuration string) error ReportProjectRun(repo string, projectName string, startedAt time.Time, endedAt time.Time, status string, command string, output string) error ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) + UploadJobArtefact(zipLocation string) (*int, *string, error) + DownloadJobArtefact(downloadTo string) (*string, error) } diff --git a/libs/backendapi/diggerapi.go b/libs/backendapi/diggerapi.go index 839c5bb7b..5ea2c0b6c 100644 --- a/libs/backendapi/diggerapi.go +++ b/libs/backendapi/diggerapi.go @@ -9,9 +9,12 @@ import ( "github.com/diggerhq/digger/libs/terraform_utils" "io" "log" + "mime" + "mime/multipart" "net/http" "net/url" "os" + "path" "path/filepath" "time" ) @@ -31,6 +34,14 @@ func (n NoopApi) ReportProjectJobStatus(repo string, projectName string, jobId s return nil, nil } +func (n NoopApi) UploadJobArtefact(zipLocation string) (*int, *string, error) { + return nil, nil, nil +} + +func (n NoopApi) DownloadJobArtefact(downloadTo string) (*string, error) { + return nil, nil +} + type DiggerApi struct { DiggerHost string AuthToken string @@ -187,6 +198,135 @@ func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId return &response, nil } +func (d DiggerApi) UploadJobArtefact(zipLocation string) (*int, *string, error) { + u, err := url.Parse(d.DiggerHost) + if err != nil { + return nil, nil, err + } + u.Path = path.Join(u.Path, "job_artefacts") + url := u.String() + filePath := zipLocation + + // Open the file + file, err := os.Open(filePath) + if err != nil { + fmt.Println("Error opening file:", err) + return nil, nil, fmt.Errorf("Error opening file:", err) + } + defer file.Close() + + // Create a buffer to store our request body as bytes + var requestBody bytes.Buffer + + // Create a multipart writer + multipartWriter := multipart.NewWriter(&requestBody) + + // Create a form file writer for our file field + fileWriter, err := multipartWriter.CreateFormFile("file", filepath.Base(filePath)) + if err != nil { + fmt.Println("Error creating form file:", err) + return nil, nil, fmt.Errorf("Error creating form file:", err) + } + + // Copy the file content to the form file writer + _, err = io.Copy(fileWriter, file) + if err != nil { + fmt.Println("Error copying file content:", err) + return nil, nil, fmt.Errorf("Error copying file content:", err) + } + + // Close the multipart writer to finalize the request body + multipartWriter.Close() + + // Create a new HTTP request + req, err := http.NewRequest("PUT", url, &requestBody) + if err != nil { + fmt.Println("Error creating request:", err) + return nil, nil, fmt.Errorf("Error creating request:", err) + } + + // Set the content type header + req.Header.Set("Content-Type", multipartWriter.FormDataContentType()) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", d.AuthToken)) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + fmt.Println("Error sending request:", err) + return nil, nil, fmt.Errorf("Error sending request:", err) + } + defer resp.Body.Close() + + // Read and print the response + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response:", err) + return nil, nil, fmt.Errorf("Error reading response: %v", err) + } + + b := string(body) + return &resp.StatusCode, &b, nil +} + +func getFilename(resp *http.Response) string { + // Check the Content-Disposition header + if cd := resp.Header.Get("Content-Disposition"); cd != "" { + if _, params, err := mime.ParseMediaType(cd); err == nil { + if filename, ok := params["filename"]; ok { + return filename + } + } + } + // Fallback to the last part of the URL path + return path.Base(resp.Request.URL.Path) +} + +func (d DiggerApi) DownloadJobArtefact(downloadTo string) (*string, error) { + // Download the zip file + url, err := url.JoinPath(d.DiggerHost, "job_artefacts") + if err != nil { + log.Printf("failed to create url: %v", err) + return nil, err + } + + // Create a new HTTP request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + fmt.Println("Error creating request:", err) + return nil, fmt.Errorf("Error creating request:", err) + } + + // Set the content type header + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", d.AuthToken)) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download zip: %w", err) + } + defer resp.Body.Close() + + // Create a temporary file to store the zip + tempZipFile, err := os.Create(path.Join(downloadTo, getFilename(resp))) + if err != nil { + return nil, fmt.Errorf("failed to create zip file: %w", err) + } + defer tempZipFile.Close() + + // Copy the downloaded content to the temporary file + _, err = io.Copy(tempZipFile, resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to save zip content: %w", err) + } + + // note that fileName include absolute path to the zip file + fileName := tempZipFile.Name() + return &fileName, nil + +} + func NewBackendApi(hostName string, authToken string) Api { var backendApi Api if os.Getenv("NO_BACKEND") == "true" { diff --git a/libs/backendapi/mocks.go b/libs/backendapi/mocks.go index 177941b3b..22b335bc7 100644 --- a/libs/backendapi/mocks.go +++ b/libs/backendapi/mocks.go @@ -20,3 +20,11 @@ func (t MockBackendApi) ReportProjectRun(repo string, projectName string, starte func (t MockBackendApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) { return nil, nil } + +func (t MockBackendApi) UploadJobArtefact(zipLocation string) (*int, *string, error) { + return nil, nil, nil +} + +func (t MockBackendApi) DownloadJobArtefact(downloadTo string) (*string, error) { + return nil, nil +} From 9ac73524d0fe4676117821c884aa8b386aeb2cfc Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 12:32:56 +0100 Subject: [PATCH 07/14] artefacts --- cli/pkg/spec/spec.go | 20 ++++++++++++-------- ee/backend/controllers/spec.go | 5 ++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index bde6057cf..7933f0674 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -5,6 +5,7 @@ import ( "github.com/diggerhq/digger/cli/pkg/digger" "github.com/diggerhq/digger/cli/pkg/usage" "github.com/diggerhq/digger/cli/pkg/utils" + "github.com/diggerhq/digger/libs/backendapi" "github.com/diggerhq/digger/libs/ci" "github.com/diggerhq/digger/libs/comment_utils/reporting" comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary" @@ -191,8 +192,9 @@ func RunSpecManualCommand( os.Chdir(gitLocation) cmd := exec.Command("git", "init") cmd.Stdout = os.Stdout - cmd.Stdout = os.Stderr - + cmd.Stderr = os.Stderr + cmd.Run() + policyChecker, err := policyProvider.GetPolicyProvider(spec.Policy, spec.Backend.BackendHostname, spec.Backend.BackendOrganisationName, spec.Backend.BackendJobToken) if err != nil { usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get policy provider: %v", err), 1) @@ -206,11 +208,13 @@ func RunSpecManualCommand( jobs := []scheduler.Job{job} - fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName) - _, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "") - if err != nil { - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4) - } + //fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName) + //_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "") + //if err != nil { + // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4) + //} + + noopBackendApi := backendapi.NoopApi{} commentId := spec.CommentId if err != nil { @@ -225,7 +229,7 @@ func RunSpecManualCommand( commentUpdater := comment_summary.NoopCommentUpdater{} // do not change these placeholders as they are parsed by dgctl to stream logs log.Printf("<========= DIGGER RUNNING IN MANUAL MODE =========>") - allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, false, false, commentId, currentDir) + allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, noopBackendApi, spec.JobId, false, false, commentId, currentDir) log.Printf("<========= DIGGER COMPLETED =========>") if err != nil || allAppliesSuccess == false { usage.ReportErrorAndExit(spec.VCS.RepoOwner, "Terraform execution failed", 1) diff --git a/ee/backend/controllers/spec.go b/ee/backend/controllers/spec.go index d78fd75eb..8ad633dd3 100644 --- a/ee/backend/controllers/spec.go +++ b/ee/backend/controllers/spec.go @@ -92,7 +92,10 @@ func (d DiggerEEController) GetSpec(c *gin.Context) { LockType: "noop", }, Backend: spec.BackendSpec{ - BackendType: "noop", + BackendType: "backend", + BackendHostname: jobSpec.BackendHostname, + BackendJobToken: jobSpec.BackendJobToken, + BackendOrganisationName: jobSpec.BackendOrganisationName, }, VCS: spec.VcsSpec{ VcsType: "github", From cf013eeec0cecb465d7c5eff426fa0fab7d879f6 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 12:46:15 +0100 Subject: [PATCH 08/14] debug logs --- cli/pkg/spec/spec.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 7933f0674..d11e0ad1c 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -189,7 +189,17 @@ func RunSpecManualCommand( utils.ExtractZip(zipPath, tempDir) // Transforming: /var/temp/xxx/yyy/blabla.zip -> /var/temp/xxx/yyy/blabla gitLocation := strings.TrimSuffix(zipPath, ".zip") - os.Chdir(gitLocation) + log.Printf("git location is: %v", gitLocation) + log.Printf("zipPath: %v", zipPath) + f, err := os.Stat(tempDir) + log.Printf("files in %v : %v ", tempDir, f) + + err = os.Chdir(gitLocation) + if err != nil { + log.Printf("Could not chdir: %v", err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not change directory: %v", err), 1) + } + cmd := exec.Command("git", "init") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr From 7ca46e17f435a548f08c942ac5e108c7cdd323bb Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 14:58:26 +0100 Subject: [PATCH 09/14] fix the cli paths --- cli/pkg/spec/spec.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index d11e0ad1c..95d9b94bf 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -17,7 +17,6 @@ import ( "log" "os" "os/exec" - "strings" "time" ) @@ -186,18 +185,25 @@ func RunSpecManualCommand( usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not download job artefact: %v", err), 1) } zipPath := *absoluteFileName - utils.ExtractZip(zipPath, tempDir) - // Transforming: /var/temp/xxx/yyy/blabla.zip -> /var/temp/xxx/yyy/blabla - gitLocation := strings.TrimSuffix(zipPath, ".zip") - log.Printf("git location is: %v", gitLocation) - log.Printf("zipPath: %v", zipPath) - f, err := os.Stat(tempDir) - log.Printf("files in %v : %v ", tempDir, f) - - err = os.Chdir(gitLocation) - if err != nil { - log.Printf("Could not chdir: %v", err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not change directory: %v", err), 1) + err = utils.ExtractZip(zipPath, tempDir) + if err != nil { + log.Printf("ExtractZip err: %v", err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("artefact zip extraction err: %v", err), 1) + + } + + // remove the zipPath + err = os.Remove(zipPath) + if err != nil { + log.Printf("os.Remove err: %v", err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("zip path removal err: %v", err), 1) + } + + // Navigating to our diractory + err = os.Chdir(tempDir) + if err != nil { + log.Printf("Chdir err: %v", err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Chdir err: %v", err), 1) } cmd := exec.Command("git", "init") From e5ba1740739846e346894e42100ffc15ecbeda56 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 14:59:03 +0100 Subject: [PATCH 10/14] no need for branch --- dgctl/cmd/exec.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index 37ed886fb..fdca3989f 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -289,7 +289,7 @@ var execCmd = &cobra.Command{ log.Printf("server response: %v", *respBody) os.Exit(1) } - + token := os.Getenv("GITHUB_PAT_TOKEN") if token == "" { log.Printf("missing variable: GITHUB_PAT_TOKEN") @@ -312,7 +312,6 @@ var execCmd = &cobra.Command{ RunName: fmt.Sprintf("digger %v manual run by %v", command, spec.VCS.Actor), } _, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), spec.VCS.RepoOwner, spec.VCS.RepoName, spec.VCS.WorkflowFile, github.CreateWorkflowDispatchEventRequest{ - Ref: spec.Job.Branch, Inputs: inputs.ToMap(), }) From 73cd4493bb3dd114f2ccd5ee28194cf2f5229cf6 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 15:27:20 +0100 Subject: [PATCH 11/14] no dependency on branches --- dgctl/cmd/exec.go | 44 ++++++++++++++++++---------------- ee/backend/controllers/spec.go | 8 +++---- libs/spec/payloads.go | 22 ++++++++--------- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index fdca3989f..51d589f3e 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -94,15 +94,13 @@ func GetUrlContents(url string) (string, error) { return content, nil } -func GetSpec(diggerUrl string, authToken string, command string, actor string, projectMarshalled string, diggerConfigMarshalled string, repoFullName string, defaultBanch string, prBranch string) ([]byte, error) { +func GetSpec(diggerUrl string, authToken string, command string, actor string, projectMarshalled string, diggerConfigMarshalled string, repoFullName string) ([]byte, error) { payload := spec.GetSpecPayload{ - Command: command, - RepoFullName: repoFullName, - Actor: actor, - DefaultBranch: defaultBanch, - PrBranch: prBranch, - DiggerConfig: diggerConfigMarshalled, - Project: projectMarshalled, + Command: command, + RepoFullName: repoFullName, + Actor: actor, + DiggerConfig: diggerConfigMarshalled, + Project: projectMarshalled, } u, err := url.Parse(diggerUrl) if err != nil { @@ -236,17 +234,17 @@ var execCmd = &cobra.Command{ os.Exit(1) } - defaultBanch, err := getDefaultBranch() - if err != nil { - log.Printf("could not get default branch, please enter manually:") - fmt.Scanln(&defaultBanch) - } + //defaultBanch, err := getDefaultBranch() + //if err != nil { + // log.Printf("could not get default branch, please enter manually:") + // fmt.Scanln(&defaultBanch) + //} - prBranch, err := getPrBranch() - if err != nil { - log.Printf("could not get current branch, please enter manually:") - fmt.Scanln(&prBranch) - } + //prBranch, err := getPrBranch() + //if err != nil { + // log.Printf("could not get current branch, please enter manually:") + // fmt.Scanln(&prBranch) + //} projectName := execConfig.Project command := execConfig.Command @@ -268,7 +266,7 @@ var execCmd = &cobra.Command{ os.Exit(1) } - specBytes, err := GetSpec(diggerHostname, "abc123", command, actor, string(projectMarshalled), string(configMarshalled), repoFullname, defaultBanch, prBranch) + specBytes, err := GetSpec(diggerHostname, "abc123", command, actor, string(projectMarshalled), string(configMarshalled), repoFullname) if err != nil { log.Printf("failed to get spec from backend: %v", err) } @@ -307,11 +305,18 @@ var execCmd = &cobra.Command{ } } + repoOwner, repoName, _ := strings.Cut(repoFullname, "/") + repository, _, err := client.Repositories.Get(context.Background(), repoOwner, repoName) + if err != nil { + log.Fatalf("Failed to get repository: %v", err) + } + inputs := orchestrator_scheduler.WorkflowInput{ Spec: string(specBytes), RunName: fmt.Sprintf("digger %v manual run by %v", command, spec.VCS.Actor), } _, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), spec.VCS.RepoOwner, spec.VCS.RepoName, spec.VCS.WorkflowFile, github.CreateWorkflowDispatchEventRequest{ + Ref: *repository.DefaultBranch, Inputs: inputs.ToMap(), }) @@ -321,7 +326,6 @@ var execCmd = &cobra.Command{ log.Printf("workflow has triggered successfully! waiting for results ...") } - repoOwner, repoName, _ := strings.Cut(repoFullname, "/") var logsUrl *string var runId *int64 var jobId *int64 diff --git a/ee/backend/controllers/spec.go b/ee/backend/controllers/spec.go index 8ad633dd3..399246b18 100644 --- a/ee/backend/controllers/spec.go +++ b/ee/backend/controllers/spec.go @@ -32,8 +32,8 @@ func (d DiggerEEController) GetSpec(c *gin.Context) { actor := payload.Actor commitSha := "" - defaultBranch := payload.DefaultBranch - prBranch := payload.PrBranch + //defaultBranch := payload.DefaultBranch + //prBranch := payload.PrBranch issueNumber := 000 config := digger_config.DiggerConfig{} @@ -54,7 +54,7 @@ func (d DiggerEEController) GetSpec(c *gin.Context) { workflowFile := project.WorkflowFile - jobs, err := generic.CreateJobsForProjects([]digger_config.Project{project}, command, "manual", repoFullName, actor, config.Workflows, &issueNumber, &commitSha, defaultBranch, prBranch) + jobs, err := generic.CreateJobsForProjects([]digger_config.Project{project}, command, "manual", repoFullName, actor, config.Workflows, &issueNumber, &commitSha, "", "") if err != nil { log.Printf("could not create jobs based on project: %v", err) c.String(500, fmt.Sprintf("could not create jobs based on project: %v", err)) @@ -77,7 +77,7 @@ func (d DiggerEEController) GetSpec(c *gin.Context) { } backendHostName := os.Getenv("HOSTNAME") - jobSpec := scheduler.JobToJson(job, scheduler.DiggerCommandPlan, org.Name, prBranch, commitSha, jobToken.Value, backendHostName, project) + jobSpec := scheduler.JobToJson(job, scheduler.DiggerCommandPlan, org.Name, "", commitSha, jobToken.Value, backendHostName, project) spec := spec.Spec{ SpecType: spec.SpecTypeManualJob, diff --git a/libs/spec/payloads.go b/libs/spec/payloads.go index 74fb22e2e..076febca1 100644 --- a/libs/spec/payloads.go +++ b/libs/spec/payloads.go @@ -1,13 +1,13 @@ package spec type GetSpecPayload struct { - Command string `json:"command"` - RepoFullName string `json:"repo_full_name"` - Actor string `json:"actor"` - DefaultBranch string `json:"default_branch"` - PrBranch string `json:"pr_branch"` - DiggerConfig string `json:"digger_config"` - Project string `json:"project"` + Command string `json:"command"` + RepoFullName string `json:"repo_full_name"` + Actor string `json:"actor"` + //DefaultBranch string `json:"default_branch"` + //PrBranch string `json:"pr_branch"` + DiggerConfig string `json:"digger_config"` + Project string `json:"project"` } func (p GetSpecPayload) ToMapStruct() map[string]interface{} { @@ -15,9 +15,9 @@ func (p GetSpecPayload) ToMapStruct() map[string]interface{} { "command": p.Command, "repo_full_name": p.RepoFullName, "actor": p.Actor, - "default_branch": p.DefaultBranch, - "pr_branch": p.PrBranch, - "digger_config": p.DiggerConfig, - "project": p.Project, + //"default_branch": p.DefaultBranch, + //"pr_branch": p.PrBranch, + "digger_config": p.DiggerConfig, + "project": p.Project, } } From b86a4fc7f746bc1c893dfb5bda87cc7ab08fc843 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 16:07:47 +0100 Subject: [PATCH 12/14] final fixes --- dgctl/cmd/exec.go | 34 +++------------ dgctl/utils/backendutils.go | 84 ------------------------------------- dgctl/utils/gitio.go | 6 ++- 3 files changed, 9 insertions(+), 115 deletions(-) delete mode 100644 dgctl/utils/backendutils.go diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index 51d589f3e..00dd94de4 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -43,18 +43,6 @@ func getRepoUsername() (string, error) { return strings.TrimSpace(string(out)), err } -func getDefaultBranch() (string, error) { - cmd := exec.Command("git", "symbolic-ref", "refs/remotes/origin/HEAD") - out, err := cmd.Output() - return strings.ReplaceAll(strings.TrimSpace(string(out)), "refs/remotes/origin/", ""), err -} - -func getPrBranch() (string, error) { - cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD") - out, err := cmd.Output() - return strings.TrimSpace(string(out)), err -} - func getRepoFullname() (string, error) { // Execute 'git config --get remote.origin.url' to get the URL of the origin remote cmd := exec.Command("git", "config", "--get", "remote.origin.url") @@ -142,11 +130,6 @@ func GetSpec(diggerUrl string, authToken string, command string, actor string, p return body, nil } -func pushToBranch(prBranch string) error { - cmd := exec.Command("git", "push", "origin", prBranch) - _, err := cmd.Output() - return err -} func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, repoName string, diggerJobID string) (*int64, *int64, *string, error) { timeFilter := time.Now().Add(-5 * time.Minute) @@ -234,18 +217,6 @@ var execCmd = &cobra.Command{ os.Exit(1) } - //defaultBanch, err := getDefaultBranch() - //if err != nil { - // log.Printf("could not get default branch, please enter manually:") - // fmt.Scanln(&defaultBanch) - //} - - //prBranch, err := getPrBranch() - //if err != nil { - // log.Printf("could not get current branch, please enter manually:") - // fmt.Scanln(&prBranch) - //} - projectName := execConfig.Project command := execConfig.Command projectConfig := config.GetProject(projectName) @@ -269,6 +240,7 @@ var execCmd = &cobra.Command{ specBytes, err := GetSpec(diggerHostname, "abc123", command, actor, string(projectMarshalled), string(configMarshalled), repoFullname) if err != nil { log.Printf("failed to get spec from backend: %v", err) + os.Exit(1) } var spec spec.Spec err = json.Unmarshal(specBytes, &spec) @@ -276,6 +248,10 @@ var execCmd = &cobra.Command{ // attach zip archive to backend backendToken := spec.Job.BackendJobToken zipLocation, err := utils.ArchiveGitRepo("./") + if err != nil { + log.Printf("error archiving zip repo: %v", err) + os.Exit(1) + } backendApi := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken} statusCode, respBody, err := backendApi.UploadJobArtefact(zipLocation) if err != nil { diff --git a/dgctl/utils/backendutils.go b/dgctl/utils/backendutils.go deleted file mode 100644 index 99182b73b..000000000 --- a/dgctl/utils/backendutils.go +++ /dev/null @@ -1,84 +0,0 @@ -package utils - -import ( - "bytes" - "fmt" - "io" - "mime/multipart" - "net/http" - "net/url" - "os" - "path" - "path/filepath" -) - -func SendZipAsJobArtefact(backendUrl string, zipLocation string, jobToken string) (*int, *string, error) { - u, err := url.Parse(backendUrl) - if err != nil { - return nil, nil, err - } - u.Path = path.Join(u.Path, "job_artefacts") - url := u.String() - filePath := zipLocation - - // Open the file - file, err := os.Open(filePath) - if err != nil { - fmt.Println("Error opening file:", err) - return nil, nil, fmt.Errorf("Error opening file:", err) - } - defer file.Close() - - // Create a buffer to store our request body as bytes - var requestBody bytes.Buffer - - // Create a multipart writer - multipartWriter := multipart.NewWriter(&requestBody) - - // Create a form file writer for our file field - fileWriter, err := multipartWriter.CreateFormFile("file", filepath.Base(filePath)) - if err != nil { - fmt.Println("Error creating form file:", err) - return nil, nil, fmt.Errorf("Error creating form file:", err) - } - - // Copy the file content to the form file writer - _, err = io.Copy(fileWriter, file) - if err != nil { - fmt.Println("Error copying file content:", err) - return nil, nil, fmt.Errorf("Error copying file content:", err) - } - - // Close the multipart writer to finalize the request body - multipartWriter.Close() - - // Create a new HTTP request - req, err := http.NewRequest("PUT", url, &requestBody) - if err != nil { - fmt.Println("Error creating request:", err) - return nil, nil, fmt.Errorf("Error creating request:", err) - } - - // Set the content type header - req.Header.Set("Content-Type", multipartWriter.FormDataContentType()) - req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", jobToken)) - - // Send the request - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - fmt.Println("Error sending request:", err) - return nil, nil, fmt.Errorf("Error sending request:", err) - } - defer resp.Body.Close() - - // Read and print the response - body, err := io.ReadAll(resp.Body) - if err != nil { - fmt.Println("Error reading response:", err) - return nil, nil, fmt.Errorf("Error reading response: %v", err) - } - - b := string(body) - return &resp.StatusCode, &b, nil -} diff --git a/dgctl/utils/gitio.go b/dgctl/utils/gitio.go index 97eeec4c4..c4f10b150 100644 --- a/dgctl/utils/gitio.go +++ b/dgctl/utils/gitio.go @@ -12,8 +12,10 @@ import ( func ArchiveGitRepo(sourcePath string) (string, error) { // Generate a unique ID for the temp directory tempID := fmt.Sprintf("%d", time.Now().UnixNano()) - tempDir := filepath.Join(os.TempDir(), fmt.Sprintf("temp_%s", tempID)) - + tempDir, err := os.MkdirTemp("", "archive-zip-") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %v", err) + } // Create the temp directory if err := os.MkdirAll(tempDir, 0755); err != nil { return "", fmt.Errorf("failed to create temp directory: %w", err) From 7fdc4e97c92fd91a0f71f1303a1c413897c0fbad Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 16:18:21 +0100 Subject: [PATCH 13/14] fix tests --- cli/pkg/spec/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 95d9b94bf..ad9f6b12b 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -175,7 +175,7 @@ func RunSpecManualCommand( // download zip artefact, git init and prepare for job execution tempDir, err := os.MkdirTemp("", "downloaded-zip-") if err != nil { - log.Printf("failed to create temp dir: %w", err) + log.Printf("failed to create temp dir: %v", err) os.Exit(1) } From 355739eed7ff2116e7e74efbc952c235307fcfc1 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 30 Jul 2024 16:24:09 +0100 Subject: [PATCH 14/14] fix libs tests --- libs/backendapi/diggerapi.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libs/backendapi/diggerapi.go b/libs/backendapi/diggerapi.go index 5ea2c0b6c..10ebc3ea0 100644 --- a/libs/backendapi/diggerapi.go +++ b/libs/backendapi/diggerapi.go @@ -189,7 +189,7 @@ func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId body, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("Could not read response body: %v", err) + return nil, fmt.Errorf("could not read response body: %v", err) } var response scheduler.SerializedBatch @@ -204,14 +204,14 @@ func (d DiggerApi) UploadJobArtefact(zipLocation string) (*int, *string, error) return nil, nil, err } u.Path = path.Join(u.Path, "job_artefacts") - url := u.String() + uploadUrl := u.String() filePath := zipLocation // Open the file file, err := os.Open(filePath) if err != nil { - fmt.Println("Error opening file:", err) - return nil, nil, fmt.Errorf("Error opening file:", err) + fmt.Println("error opening file:", err) + return nil, nil, fmt.Errorf("error opening file: %v", err) } defer file.Close() @@ -225,24 +225,24 @@ func (d DiggerApi) UploadJobArtefact(zipLocation string) (*int, *string, error) fileWriter, err := multipartWriter.CreateFormFile("file", filepath.Base(filePath)) if err != nil { fmt.Println("Error creating form file:", err) - return nil, nil, fmt.Errorf("Error creating form file:", err) + return nil, nil, fmt.Errorf("error creating form file: %v", err) } // Copy the file content to the form file writer _, err = io.Copy(fileWriter, file) if err != nil { fmt.Println("Error copying file content:", err) - return nil, nil, fmt.Errorf("Error copying file content:", err) + return nil, nil, fmt.Errorf("error copying file content: %v", err) } // Close the multipart writer to finalize the request body multipartWriter.Close() // Create a new HTTP request - req, err := http.NewRequest("PUT", url, &requestBody) + req, err := http.NewRequest("PUT", uploadUrl, &requestBody) if err != nil { fmt.Println("Error creating request:", err) - return nil, nil, fmt.Errorf("Error creating request:", err) + return nil, nil, fmt.Errorf("error creating request: %v", err) } // Set the content type header @@ -254,7 +254,7 @@ func (d DiggerApi) UploadJobArtefact(zipLocation string) (*int, *string, error) resp, err := client.Do(req) if err != nil { fmt.Println("Error sending request:", err) - return nil, nil, fmt.Errorf("Error sending request:", err) + return nil, nil, fmt.Errorf("error sending request: %v", err) } defer resp.Body.Close() @@ -262,7 +262,7 @@ func (d DiggerApi) UploadJobArtefact(zipLocation string) (*int, *string, error) body, err := io.ReadAll(resp.Body) if err != nil { fmt.Println("Error reading response:", err) - return nil, nil, fmt.Errorf("Error reading response: %v", err) + return nil, nil, fmt.Errorf("error reading response: %v", err) } b := string(body) @@ -284,17 +284,17 @@ func getFilename(resp *http.Response) string { func (d DiggerApi) DownloadJobArtefact(downloadTo string) (*string, error) { // Download the zip file - url, err := url.JoinPath(d.DiggerHost, "job_artefacts") + downloadUrl, err := url.JoinPath(d.DiggerHost, "job_artefacts") if err != nil { log.Printf("failed to create url: %v", err) return nil, err } // Create a new HTTP request - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest("GET", downloadUrl, nil) if err != nil { fmt.Println("Error creating request:", err) - return nil, fmt.Errorf("Error creating request:", err) + return nil, fmt.Errorf("error creating request: %v", err) } // Set the content type header