From 47f3163981b6a2cbd78c1a5003d613ce9546b99b Mon Sep 17 00:00:00 2001 From: pransh62390 <63577123+pransh62390@users.noreply.github.com> Date: Fri, 24 Jan 2025 01:08:02 +0530 Subject: [PATCH 1/3] added stage in spec and solved issue of checking duplicates of multiple stages solves issue: #1795 --- backend/services/spec.go | 51 ++++++++++++++++++++++++++++++---------- libs/spec/models.go | 1 + 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/backend/services/spec.go b/backend/services/spec.go index 152f7199c..986ab60eb 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -60,7 +60,7 @@ func GetRunNameFromJob(job models.DiggerJob) (*string, error) { return &runName, nil } -func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec { +func getVariablesSpecFromEnvMap(envVars map[string]string, stage string) []spec.VariableSpec { variablesSpec := make([]spec.VariableSpec, 0) for k, v := range envVars { if strings.HasPrefix(v, "$DIGGER_") { @@ -70,6 +70,7 @@ func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec { Value: val, IsSecret: false, IsInterpolated: true, + Stage: stage, }) } else { variablesSpec = append(variablesSpec, spec.VariableSpec{ @@ -77,6 +78,7 @@ func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec { Value: v, IsSecret: false, IsInterpolated: false, + Stage: stage, }) } @@ -84,6 +86,27 @@ func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec { return variablesSpec } +func findDuplicatesInStage(variablesSpec []spec.VariableSpec, stage string) (error) { + // Extract the names from VariableSpec + justNames := lo.Map(variablesSpec, func(item VariableSpec, i int) string { + return item.Name + }) + + // Group names by their occurrence + nameCounts := lo.CountValues(justNames) + + // Filter names that occur more than once + duplicates := lo.Keys(lo.Filter(nameCounts, func(count int, name string) bool { + return count > 1 + })) + + if len(duplicates) > 0 { + return fmt.Errorf("In %v stage, found duplicate variables: %v", stage, duplicates) + } + + return nil // No duplicates found +} + func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { var jobSpec scheduler.JobJson err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec) @@ -92,23 +115,25 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { return nil, fmt.Errorf("could not marshal json string: %v", err) } + stateVariables := getVariablesSpecFromEnvMap(jobSpec.StateEnvVars, "state") + commandVariables := getVariablesSpecFromEnvMap(jobSpec.CommandEnvVars, "commands") + runVariables := getVariablesSpecFromEnvMap(jobSpec.RunEnvVars, "run") + + if err := checkDuplicatesInStage(stateVariables, "state"); err != nil { + return nil, err + } + if err := checkDuplicatesInStage(commandVariables, "commands"); err != nil { + return nil, err + } + if err := checkDuplicatesInStage(runVariables, "run"); err != nil { + return nil, err + } + variablesSpec := make([]spec.VariableSpec, 0) - stateVariables := getVariablesSpecFromEnvMap(jobSpec.StateEnvVars) - commandVariables := getVariablesSpecFromEnvMap(jobSpec.CommandEnvVars) - runVariables := getVariablesSpecFromEnvMap(jobSpec.RunEnvVars) variablesSpec = append(variablesSpec, stateVariables...) variablesSpec = append(variablesSpec, commandVariables...) variablesSpec = append(variablesSpec, runVariables...) - // check for duplicates in list of variablesSpec - justNames := lo.Map(variablesSpec, func(item spec.VariableSpec, i int) string { - return item.Name - }) - hasDuplicates := len(justNames) != len(lo.Uniq(justNames)) - if hasDuplicates { - return nil, fmt.Errorf("could not load variables due to duplicates") - } - batch := job.Batch spec := spec.Spec{ diff --git a/libs/spec/models.go b/libs/spec/models.go index 3637f920b..6818e1ed3 100644 --- a/libs/spec/models.go +++ b/libs/spec/models.go @@ -45,6 +45,7 @@ type VariableSpec struct { Value string `json:"value"` IsSecret bool `json:"is_secret"` IsInterpolated bool `json:"is_interpolated"` + Stage string `json:"stage"` } type SpecType string From 34e2b2a3a290260e363c6827efbc810efc30c518 Mon Sep 17 00:00:00 2001 From: pransh62390 <63577123+pransh62390@users.noreply.github.com> Date: Fri, 24 Jan 2025 01:24:22 +0530 Subject: [PATCH 2/3] fixed errors and error logging message --- backend/services/spec.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/services/spec.go b/backend/services/spec.go index 986ab60eb..725f36085 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -88,7 +88,7 @@ func getVariablesSpecFromEnvMap(envVars map[string]string, stage string) []spec. func findDuplicatesInStage(variablesSpec []spec.VariableSpec, stage string) (error) { // Extract the names from VariableSpec - justNames := lo.Map(variablesSpec, func(item VariableSpec, i int) string { + justNames := lo.Map(variablesSpec, func(item spec.VariableSpec, i int) string { return item.Name }) @@ -101,7 +101,7 @@ func findDuplicatesInStage(variablesSpec []spec.VariableSpec, stage string) (err })) if len(duplicates) > 0 { - return fmt.Errorf("In %v stage, found duplicate variables: %v", stage, duplicates) + return fmt.Errorf("duplicate variable names found in '%s' stage: %v", stage, strings.Join(duplicates, ", ")) } return nil // No duplicates found @@ -119,13 +119,13 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) { commandVariables := getVariablesSpecFromEnvMap(jobSpec.CommandEnvVars, "commands") runVariables := getVariablesSpecFromEnvMap(jobSpec.RunEnvVars, "run") - if err := checkDuplicatesInStage(stateVariables, "state"); err != nil { + if err := findDuplicatesInStage(stateVariables, "state"); err != nil { return nil, err } - if err := checkDuplicatesInStage(commandVariables, "commands"); err != nil { + if err := findDuplicatesInStage(commandVariables, "commands"); err != nil { return nil, err } - if err := checkDuplicatesInStage(runVariables, "run"); err != nil { + if err := findDuplicatesInStage(runVariables, "run"); err != nil { return nil, err } From 318babd00ae007d19da3e9578a063be6b31ef74c Mon Sep 17 00:00:00 2001 From: pransh62390 <63577123+pransh62390@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:52:34 +0530 Subject: [PATCH 3/3] fixed compilation issue --- backend/services/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/spec.go b/backend/services/spec.go index 725f36085..590473fef 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -96,7 +96,7 @@ func findDuplicatesInStage(variablesSpec []spec.VariableSpec, stage string) (err nameCounts := lo.CountValues(justNames) // Filter names that occur more than once - duplicates := lo.Keys(lo.Filter(nameCounts, func(count int, name string) bool { + duplicates := lo.Keys(lo.PickBy(nameCounts, func(name string, count int) bool { return count > 1 }))