From a91951e403b7a9ed79d042b1855170d10f9ebab4 Mon Sep 17 00:00:00 2001 From: Mohamed Habib Date: Thu, 29 Aug 2024 03:22:10 -0700 Subject: [PATCH] feat/unify/runspec (#1683) --- backend/services/spec.go | 5 +- cli/cmd/digger/default.go | 41 ++---- cli/pkg/spec/next.go | 180 ------------------------- cli/pkg/spec/spec.go | 37 ++++- libs/comment_utils/summary/provider.go | 3 +- libs/spec/models.go | 22 +-- next/services/runs.go | 5 +- next/services/scheduler.go | 11 +- next/services/spec.go | 11 +- 9 files changed, 80 insertions(+), 235 deletions(-) delete mode 100644 cli/pkg/spec/next.go diff --git a/backend/services/spec.go b/backend/services/spec.go index 958b11890..71ac799dd 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/digger_config" "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/libs/spec" "github.com/samber/lo" @@ -111,7 +112,6 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { batch := job.Batch spec := spec.Spec{ - SpecType: spec.SpecTypeApplyBeforeMergeJob, JobId: job.DiggerJobID, CommentId: strconv.FormatInt(*batch.CommentId, 10), Job: jobSpec, @@ -140,6 +140,9 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { Policy: spec.PolicySpec{ PolicyType: "http", }, + CommentUpdater: spec.CommentUpdaterSpec{ + CommentUpdaterType: digger_config.CommentRenderModeBasic, + }, } return &spec, nil } diff --git a/cli/cmd/digger/default.go b/cli/cmd/digger/default.go index 92a7665f4..72f9908ec 100644 --- a/cli/cmd/digger/default.go +++ b/cli/cmd/digger/default.go @@ -31,33 +31,20 @@ var defaultCmd = &cobra.Command{ } var spec_err error - if spec.SpecType == lib_spec.SpecTypeNextJob { - spec_err = spec2.RunSpecNext( - spec, - lib_spec.VCSProviderBasic{}, - lib_spec.JobSpecProvider{}, - lib_spec.LockProvider{}, - lib_spec.ReporterProvider{}, - lib_spec.BackendApiProvider{}, - lib_spec.BasicPolicyProvider{}, - lib_spec.PlanStorageProvider{}, - lib_spec.VariablesProvider{}, - comment_updater.CommentUpdaterProviderBasic{}, - ) - } else { - spec_err = spec2.RunSpec( - spec, - lib_spec.VCSProviderBasic{}, - lib_spec.JobSpecProvider{}, - lib_spec.LockProvider{}, - lib_spec.ReporterProvider{}, - lib_spec.BackendApiProvider{}, - lib_spec.BasicPolicyProvider{}, - lib_spec.PlanStorageProvider{}, - lib_spec.VariablesProvider{}, - comment_updater.CommentUpdaterProviderBasic{}, - ) - } + + spec_err = spec2.RunSpec( + spec, + lib_spec.VCSProviderBasic{}, + lib_spec.JobSpecProvider{}, + lib_spec.LockProvider{}, + lib_spec.ReporterProvider{}, + lib_spec.BackendApiProvider{}, + lib_spec.BasicPolicyProvider{}, + lib_spec.PlanStorageProvider{}, + lib_spec.VariablesProvider{}, + comment_updater.CommentUpdaterProviderBasic{}, + ) + if spec_err != nil { usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("error while running spec: %v", err), 1) } diff --git a/cli/pkg/spec/next.go b/cli/pkg/spec/next.go deleted file mode 100644 index 968d20a77..000000000 --- a/cli/pkg/spec/next.go +++ /dev/null @@ -1,180 +0,0 @@ -package spec - -import ( - "fmt" - "github.com/diggerhq/digger/cli/pkg/digger" - "github.com/diggerhq/digger/cli/pkg/usage" - backend2 "github.com/diggerhq/digger/libs/backendapi" - "github.com/diggerhq/digger/libs/ci" - "github.com/diggerhq/digger/libs/comment_utils/reporting" - comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary" - "github.com/diggerhq/digger/libs/scheduler" - "github.com/diggerhq/digger/libs/spec" - "github.com/samber/lo" - "log" - "os" - "os/exec" - "time" -) - -func reporterError(spec spec.Spec, backendApi backend2.Api, err error) { - _, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "") - if reportingError != nil { - usage.ReportErrorAndExit(spec.VCS.RepoOwner, fmt.Sprintf("Failed to run commands. %v", err), 5) - } -} - -func RunSpecNext( - spec spec.Spec, - vcsProvider spec.VCSProvider, - jobProvider spec.JobSpecProvider, - lockProvider spec.LockProvider, - reporterProvider spec.ReporterProvider, - backedProvider spec.BackendApiProvider, - policyProvider spec.SpecPolicyProvider, - PlanStorageProvider spec.PlanStorageProvider, - VariablesProvider spec.VariablesProvider, - commentUpdaterProvider comment_summary.CommentUpdaterProvider, -) error { - - backendApi, err := backedProvider.GetBackendApi(spec.Backend) - if err != nil { - log.Printf("error getting backend api: %v", err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1) - } - - // checking out to the commit ID - log.Printf("fetching commit ID %v", spec.Job.Commit) - fetchCmd := exec.Command("git", "fetch", "origin", spec.Job.Commit) - fetchCmd.Stdout = os.Stdout - fetchCmd.Stderr = os.Stderr - err = fetchCmd.Run() - if err != nil { - log.Printf("error while fetching commit SHA: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("error while checking out to commit sha: %v", err), 1) - } - - log.Printf("checking out to commit ID %v", spec.Job.Commit) - cmd := exec.Command("git", "checkout", spec.Job.Commit) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - log.Printf("error while checking out to commit SHA: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("error while checking out to commit sha: %v", err), 1) - } - - job, err := jobProvider.GetJob(spec.Job) - if err != nil { - log.Printf("error getting job: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get job: %v", err), 1) - } - - // get variables from the variables spec - variablesMap, err := VariablesProvider.GetVariables(spec.Variables) - if err != nil { - log.Printf("could not get variables from provider: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get variables from provider: %v", err), 1) - } - job.StateEnvVars = lo.Assign(job.StateEnvVars, variablesMap) - job.CommandEnvVars = lo.Assign(job.CommandEnvVars, variablesMap) - - lock, err := lockProvider.GetLock(spec.Lock) - if err != nil { - log.Printf("error getting lock: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get lock: %v", err), 1) - } - - prService := ci.MockPullRequestManager{} - //prService, err := vcsProvider.GetPrService(spec.VCS) - //if err != nil { - // log.Printf("error getting prservice: %v", err) - // reporterError(spec, backendApi, err) - // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get prservice: %v", err), 1) - //} - - orgService := ci.MockPullRequestManager{} - //orgService, err := vcsProvider.GetOrgService(spec.VCS) - //if err != nil { - // log.Printf("error getting orgservice: %v", err) - // reporterError(spec, backendApi, err) - // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get orgservice: %v", err), 1) - //} - - //reporter, err := reporterProvider.GetReporter(fmt.Sprintf("%v for %v", spec.Job.JobType, job.ProjectName), spec.Reporter, prService, *spec.Job.PullRequestNumber) - //if err != nil { - // log.Printf("error getting reporter: %v", err) - // reporterError(spec, backendApi, err) - // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get reporter: %v", err), 1) - //} - reporter := reporting.NoopReporter{} - - policyChecker, err := policyProvider.GetPolicyProvider(spec.Policy, spec.Backend.BackendHostname, spec.Backend.BackendOrganisationName, spec.Backend.BackendJobToken) - if err != nil { - log.Printf("error getting policy provider: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get policy provider: %v", err), 1) - } - - //changedFiles, err := prService.GetChangedFiles(*spec.Job.PullRequestNumber) - //if err != nil { - // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get changed files: %v", err), 1) - //} - //diggerConfig, _, _, err := digger_config.LoadDiggerConfig("./", false, changedFiles) - //if err != nil { - // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to read Digger digger_config. %s", err), 4) - //} - //log.Printf("Digger digger_config read successfully\n") - - commentUpdater := comment_summary.NoopCommentUpdater{} - //commentUpdater, err := commentUpdaterProvider.Get(*diggerConfig) - //if err != nil { - // usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get comment updater: %v", err), 8) - //} - - planStorage, err := PlanStorageProvider.GetPlanStorage(spec.VCS.RepoOwner, spec.VCS.RepoName, *spec.Job.PullRequestNumber) - if err != nil { - log.Printf("error getting plan storage: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get plan storage: %v", err), 8) - } - - jobs := []scheduler.Job{job} - - fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName) - _, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "") - if err != nil { - log.Printf("error getting project status: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4) - } - - commentId := spec.CommentId - - currentDir, err := os.Getwd() - if err != nil { - log.Printf("error getting current directory: %v", err) - reporterError(spec, backendApi, err) - usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to get current dir. %s", err), 4) - } - - reportTerraformOutput := spec.Reporter.ReportTerraformOutput - allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, true, reportTerraformOutput, commentId, currentDir) - if !allAppliesSuccess || err != nil { - _, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "") - if reportingError != nil { - usage.ReportErrorAndExit(spec.VCS.RepoOwner, fmt.Sprintf("Failed run commands. %v", err), 5) - } - //commentUpdater.UpdateComment(serializedBatch.Jobs, serializedBatch.PrNumber, prService, commentId) - //digger.UpdateAggregateStatus(serializedBatch, prService) - usage.ReportErrorAndExit(spec.VCS.RepoOwner, fmt.Sprintf("Failed to run commands. %v", err), 5) - } - usage.ReportErrorAndExit(spec.VCS.RepoOwner, "Digger finished successfully", 0) - - return nil -} diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 3e5e78618..ad9d3d7ef 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -6,12 +6,12 @@ import ( "github.com/diggerhq/digger/cli/pkg/usage" backend2 "github.com/diggerhq/digger/libs/backendapi" comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary" - "github.com/diggerhq/digger/libs/digger_config" "github.com/diggerhq/digger/libs/scheduler" "github.com/diggerhq/digger/libs/spec" "github.com/samber/lo" "log" "os" + "os/exec" "time" ) @@ -43,6 +43,31 @@ func RunSpec( usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1) } + if spec.Job.Commit != "" { + // checking out to the commit ID + log.Printf("fetching commit ID %v", spec.Job.Commit) + fetchCmd := exec.Command("git", "fetch", "origin", spec.Job.Commit) + fetchCmd.Stdout = os.Stdout + fetchCmd.Stderr = os.Stderr + err = fetchCmd.Run() + if err != nil { + msg := fmt.Sprintf("error while fetching commit SHA: %v", err) + reportError(spec, backendApi, msg, err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("error while checking out to commit sha: %v", err), 1) + } + + log.Printf("checking out to commit ID %v", spec.Job.Commit) + cmd := exec.Command("git", "checkout", spec.Job.Commit) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + msg := fmt.Sprintf("error while checking out to commit SHA: %v", err) + reportError(spec, backendApi, msg, err) + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("error while checking out to commit sha: %v", err), 1) + } + } + job, err := jobProvider.GetJob(spec.Job) if err != nil { message := fmt.Sprintf("could not get job: %v", err) @@ -79,8 +104,8 @@ func RunSpec( reportError(spec, backendApi, message, err) } - // TODO: render mode being passable from the string - commentUpdater, err := commentUpdaterProvider.Get(digger_config.CommentRenderModeBasic) + // TODO: render mode being passable from the spec as a string + commentUpdater, err := commentUpdaterProvider.Get(spec.CommentUpdater.CommentUpdaterType) if err != nil { message := fmt.Sprintf("could not get comment updater: %v", err) reportError(spec, backendApi, message, err) @@ -92,13 +117,11 @@ func RunSpec( reportError(spec, backendApi, message, err) } - // TODO: make this part purely based on variables providers - // get variables from the variables spec variablesMap, err := variablesProvider.GetVariables(spec.Variables) if err != nil { - log.Printf("could not get variables from provider: %v", err) - reporterError(spec, backendApi, err) + msg := fmt.Sprintf("could not get variables from provider: %v", err) + reportError(spec, backendApi, msg, err) usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get variables from provider: %v", err), 1) } job.StateEnvVars = lo.Assign(job.StateEnvVars, variablesMap) diff --git a/libs/comment_utils/summary/provider.go b/libs/comment_utils/summary/provider.go index 9568d2a50..1d4b8d452 100644 --- a/libs/comment_utils/summary/provider.go +++ b/libs/comment_utils/summary/provider.go @@ -15,9 +15,10 @@ func (c CommentUpdaterProviderBasic) Get(renderMode string) (CommentUpdater, err if renderMode == digger_config.CommentRenderModeBasic { return BasicCommentUpdater{}, nil } else if renderMode == digger_config.CommentRenderModeGroupByModule { - commentUpdater := BasicCommentUpdater{} return commentUpdater, nil + } else if renderMode == "noop" { + return NoopCommentUpdater{}, nil } else { return nil, fmt.Errorf("Unknown comment render mode found: %v", renderMode) } diff --git a/libs/spec/models.go b/libs/spec/models.go index 1c2dfb107..3637f920b 100644 --- a/libs/spec/models.go +++ b/libs/spec/models.go @@ -10,6 +10,10 @@ type ReporterSpec struct { ReportTerraformOutput bool `json:"report_terraform_output"` } +type CommentUpdaterSpec struct { + CommentUpdaterType string `json:"comment_updater_type"` +} + type LockSpec struct { LockType string `json:"lock_type"` LockProvider string `json:"lock_provider"` @@ -45,10 +49,7 @@ type VariableSpec struct { type SpecType string -const SpecTypeNextJob SpecType = "next_run_spec" -const SpecTypeApplyBeforeMergeJob SpecType = "before_merge_spec" const SpecTypeManualJob SpecType = "manual_job" -const SpecTypeDriftJob SpecType = "drift_job" type Spec struct { // TODO: replace these three to be nested into one of the other specs @@ -57,11 +58,12 @@ type Spec struct { CommentId string `json:"comment_id"` RunName string `json:"run_name"` - Job scheduler.JobJson `json:"job"` - Reporter ReporterSpec `json:"reporter"` - Lock LockSpec `json:"lock"` - Backend BackendSpec `json:"backend"` - VCS VcsSpec `json:"vcs"` - Policy PolicySpec `json:"policy_provider"` - Variables []VariableSpec `json:"variables"` + Job scheduler.JobJson `json:"job"` + Reporter ReporterSpec `json:"reporter"` + CommentUpdater CommentUpdaterSpec `json:"comment_updater"` + Lock LockSpec `json:"lock"` + Backend BackendSpec `json:"backend"` + VCS VcsSpec `json:"vcs"` + Policy PolicySpec `json:"policy_provider"` + Variables []VariableSpec `json:"variables"` } diff --git a/next/services/runs.go b/next/services/runs.go index 92bacbe45..d17a08814 100644 --- a/next/services/runs.go +++ b/next/services/runs.go @@ -5,7 +5,6 @@ import ( "github.com/diggerhq/digger/libs/ci" "github.com/diggerhq/digger/libs/ci/github" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" - lib_spec "github.com/diggerhq/digger/libs/spec" "github.com/diggerhq/digger/next/ci_backends" "github.com/diggerhq/digger/next/dbmodels" "github.com/diggerhq/digger/next/model" @@ -57,7 +56,7 @@ func RunQueuesStateMachine(queueItem *model.DiggerRunQueueItem, service ci.PullR return fmt.Errorf("could not get variable spec from job: %v", err) } - spec, err := GetSpecFromJob(*planJob, lib_spec.SpecTypeNextJob) + spec, err := GetSpecFromJob(*planJob) if err != nil { log.Printf("could not get spec: %v", err) return fmt.Errorf("could not get spec: %v", err) @@ -140,7 +139,7 @@ func RunQueuesStateMachine(queueItem *model.DiggerRunQueueItem, service ci.PullR return fmt.Errorf("could not get run name: %v", err) } - spec, err := GetSpecFromJob(*job, lib_spec.SpecTypeNextJob) + spec, err := GetSpecFromJob(*job) if err != nil { log.Printf("could not get spec: %v", err) return fmt.Errorf("could not get spec: %v", err) diff --git a/next/services/scheduler.go b/next/services/scheduler.go index 14a1eb456..dbff9c362 100644 --- a/next/services/scheduler.go +++ b/next/services/scheduler.go @@ -3,6 +3,7 @@ package services import ( "fmt" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/digger_config" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" lib_spec "github.com/diggerhq/digger/libs/spec" "github.com/diggerhq/digger/next/ci_backends" @@ -76,11 +77,19 @@ func TriggerJob(gh utils.GithubClientProvider, ciBackend ci_backends.CiBackend, return fmt.Errorf("could not get variable spec from job: %v", err) } - spec, err := GetSpecFromJob(*job, lib_spec.SpecTypeApplyBeforeMergeJob) + spec, err := GetSpecFromJob(*job) if err != nil { log.Printf("could not get spec: %v", err) return fmt.Errorf("could not get spec %v", err) } + // temp: overriding the reporter spec + spec.Reporter = lib_spec.ReporterSpec{ + ReportingStrategy: "comments_per_run", + ReporterType: "lazy", + ReportTerraformOutput: true, + } + // temp: overriding the comment updater type for PR + spec.CommentUpdater = lib_spec.CommentUpdaterSpec{CommentUpdaterType: digger_config.CommentRenderModeBasic} vcsToken, err := GetVCSTokenFromJob(*job, gh) if err != nil { diff --git a/next/services/spec.go b/next/services/spec.go index be1c22d8c..d128da51b 100644 --- a/next/services/spec.go +++ b/next/services/spec.go @@ -147,7 +147,7 @@ func GetRunNameFromJob(job model.DiggerJob) (*string, error) { return &runName, nil } -func GetSpecFromJob(job model.DiggerJob, specType spec.SpecType) (*spec.Spec, error) { +func GetSpecFromJob(job model.DiggerJob) (*spec.Spec, error) { var jobSpec scheduler.JobJson err := json.Unmarshal(job.JobSpec, &jobSpec) if err != nil { @@ -171,13 +171,11 @@ func GetSpecFromJob(job model.DiggerJob, specType spec.SpecType) (*spec.Spec, er } spec := spec.Spec{ - SpecType: specType, JobId: job.DiggerJobID, CommentId: strconv.FormatInt(batch.CommentID, 10), Job: jobSpec, Reporter: spec.ReporterSpec{ - ReportingStrategy: "comments_per_run", - ReporterType: "lazy", + ReporterType: "noop", ReportTerraformOutput: true, }, Lock: spec.LockSpec{ @@ -191,7 +189,7 @@ func GetSpecFromJob(job model.DiggerJob, specType spec.SpecType) (*spec.Spec, er }, Variables: variablesSpec, VCS: spec.VcsSpec{ - VcsType: string(batch.Vcs), + VcsType: batch.Vcs, Actor: jobSpec.RequestedBy, RepoFullname: batch.RepoFullName, RepoOwner: batch.RepoOwner, @@ -201,6 +199,9 @@ func GetSpecFromJob(job model.DiggerJob, specType spec.SpecType) (*spec.Spec, er Policy: spec.PolicySpec{ PolicyType: "http", }, + CommentUpdater: spec.CommentUpdaterSpec{ + CommentUpdaterType: "noop", + }, } return &spec, nil }