Skip to content

Commit

Permalink
improve efficiency in terragrunt generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@motatoes
motatoes committed Dec 12, 2024
1 parent 24badb0 commit 1ca2067
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 32 deletions.
9 changes: 3 additions & 6 deletions libs/digger_config/digger_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
CreateProjectName: true,
DefaultWorkflow: workflow,
WorkflowFile: b.WorkflowFile,
FilterPath: path.Join(terraformDir, *b.RootDir),
FilterPaths: []string{path.Join(terraformDir, *b.RootDir)},
AwsRoleToAssume: b.AwsRoleToAssume,
AwsCognitoOidcConfig: b.AwsCognitoOidcConfig,
}
Expand Down Expand Up @@ -522,7 +522,7 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
projectExternalChilds,
parsingConfig.AutoMerge,
parallel,
parsingConfig.FilterPath,
parsingConfig.FilterPaths,
parsingConfig.CreateHclProjectChilds,
ignoreParentTerragrunt,
parsingConfig.IgnoreDependencyBlocks,
Expand All @@ -536,6 +536,7 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
parsingConfig.PreserveProjects,
parsingConfig.UseProjectMarkers,
executionOrderGroups,
parsingConfig.TriggerProjectsFromDirOnly,
)
if err != nil {
return fmt.Errorf("failed to autogenerate digger_config, error during parse: %v", err)
Expand All @@ -562,10 +563,6 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
projectDir := path.Join(pathPrefix, atlantisProject.Dir)
atlantisProject.Autoplan.WhenModified, err = GetPatternsRelativeToRepo(projectDir, atlantisProject.Autoplan.WhenModified)

if parsingConfig.TriggerProjectsFromDirOnly {
atlantisProject.Autoplan.WhenModified, err = FilterPathsOutsideOfProjectPath(projectDir, atlantisProject.Autoplan.WhenModified)
}

if err != nil {
return fmt.Errorf("could not normalize patterns: %v", err)
}
Expand Down
72 changes: 47 additions & 25 deletions libs/digger_config/terragrunt/atlantis/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ func getDependencies(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, g
}

// Creates an AtlantisProject for a directory
func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, gitRoot string, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, sourcePath string) (*AtlantisProject, []string, error) {
func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, gitRoot string, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, sourcePath string, triggerProjectsFromDirOnly bool) (*AtlantisProject, []string, error) {
options, err := options.NewTerragruntOptionsWithConfigPath(sourcePath)

var potentialProjectDependencies []string
Expand All @@ -324,6 +324,39 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
options.RunTerragrunt = terraform.Run
options.Env = getEnvs()

// All dependencies depend on their own .hcl file, and any tf files in their directory
relativeDependencies := []string{
"*.hcl",
"*.tf*",
}

// Clean up the relative path to the format Atlantis expects
absoluteSourceDir := filepath.Dir(sourcePath) + string(filepath.Separator)
relativeSourceDir := strings.TrimPrefix(absoluteSourceDir, gitRoot)
relativeSourceDir = strings.TrimSuffix(relativeSourceDir, string(filepath.Separator))
if relativeSourceDir == "" {
relativeSourceDir = "."
}

if triggerProjectsFromDirOnly {
project := &AtlantisProject{
Dir: filepath.ToSlash(relativeSourceDir),
Workflow: defaultWorkflow,
TerraformVersion: defaultTerraformVersion,
ApplyRequirements: &defaultApplyRequirements,
Autoplan: AutoplanConfig{
Enabled: autoPlan,
WhenModified: uniqueStrings(relativeDependencies),
},
}
projectName := projectNameFromDir(project.Dir)

if createProjectName {
project.Name = projectName
}
return project, potentialProjectDependencies, nil
}

dependencies, err := getDependencies(ignoreParentTerragrunt, ignoreDependencyBlocks, gitRoot, cascadeDependencies, sourcePath, options)
if err != nil {
return nil, potentialProjectDependencies, err
Expand All @@ -334,8 +367,6 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
return nil, potentialProjectDependencies, nil
}

absoluteSourceDir := filepath.Dir(sourcePath) + string(filepath.Separator)

locals, err := parseLocals(sourcePath, options, nil)
if err != nil {
return nil, potentialProjectDependencies, err
Expand All @@ -346,12 +377,6 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
return nil, potentialProjectDependencies, nil
}

// All dependencies depend on their own .hcl file, and any tf files in their directory
relativeDependencies := []string{
"*.hcl",
"*.tf*",
}

// Add other dependencies based on their relative paths. We always want to output with Unix path separators
for _, dependencyPath := range dependencies {
absolutePath := dependencyPath
Expand All @@ -368,13 +393,6 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
relativeDependencies = append(relativeDependencies, filepath.ToSlash(relativePath))
}

// Clean up the relative path to the format Atlantis expects
relativeSourceDir := strings.TrimPrefix(absoluteSourceDir, gitRoot)
relativeSourceDir = strings.TrimSuffix(relativeSourceDir, string(filepath.Separator))
if relativeSourceDir == "" {
relativeSourceDir = "."
}

workflow := defaultWorkflow
if locals.AtlantisWorkflow != "" {
workflow = locals.AtlantisWorkflow
Expand Down Expand Up @@ -580,7 +598,7 @@ func createHclProject(defaultWorkflow string, defaultApplyRequirements []string,
}

// Finds the absolute paths of all terragrunt.hcl files
func getAllTerragruntFiles(filterPath string, projectHclFiles []string, path string) ([]string, error) {
func getAllTerragruntFiles(filterPaths []string, projectHclFiles []string, path string) ([]string, error) {
options, err := options.NewTerragruntOptionsWithConfigPath(path)
if err != nil {
return nil, err
Expand All @@ -592,11 +610,15 @@ func getAllTerragruntFiles(filterPath string, projectHclFiles []string, path str
workingPaths := []string{path}

// filters are not working (yet) if using project hcl files (which are kind of filters by themselves)
if filterPath != "" && len(projectHclFiles) == 0 {
// get all matching folders
workingPaths, err = filepath.Glob(filterPath)
if err != nil {
return nil, err
if len(filterPaths) > 0 && len(projectHclFiles) == 0 {
workingPaths = []string{}
for _, filterPath := range filterPaths {
// get all matching folders
theseWorkingPaths, err := filepath.Glob(filterPath)
if err != nil {
return nil, err
}
workingPaths = append(workingPaths, theseWorkingPaths...)
}
}

Expand Down Expand Up @@ -660,7 +682,7 @@ func getAllTerragruntProjectHclFiles(projectHclFiles []string, gitRoot string) m
return uniqueHclFileAbsPaths
}

func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChilds bool, autoMerge bool, parallel bool, filterPath string, createHclProjectChilds bool, ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, preserveProjects bool, useProjectMarkers bool, executionOrderGroups bool) (*AtlantisConfig, map[string][]string, error) {
func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChilds bool, autoMerge bool, parallel bool, filterPaths []string, createHclProjectChilds bool, ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, preserveProjects bool, useProjectMarkers bool, executionOrderGroups bool, triggerProjectsFromDirOnly bool) (*AtlantisConfig, map[string][]string, error) {
// Ensure the gitRoot has a trailing slash and is an absolute path
absoluteGitRoot, err := filepath.Abs(gitRoot)
if err != nil {
Expand Down Expand Up @@ -696,7 +718,7 @@ func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChi
sem := semaphore.NewWeighted(10)
projectDependenciesMap := sync.Map{}
for _, workingDir := range workingDirs {
terragruntFiles, err := getAllTerragruntFiles(filterPath, projectHclFiles, workingDir)
terragruntFiles, err := getAllTerragruntFiles(filterPaths, projectHclFiles, workingDir)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -726,7 +748,7 @@ func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChi

errGroup.Go(func() error {
defer sem.Release(1)
project, projDeps, err := createProject(ignoreParentTerragrunt, ignoreDependencyBlocks, gitRoot, cascadeDependencies, defaultWorkflow, defaultApplyRequirements, autoPlan, defaultTerraformVersion, createProjectName, createWorkspace, terragruntPath)
project, projDeps, err := createProject(ignoreParentTerragrunt, ignoreDependencyBlocks, gitRoot, cascadeDependencies, defaultWorkflow, defaultApplyRequirements, autoPlan, defaultTerraformVersion, createProjectName, createWorkspace, terragruntPath, triggerProjectsFromDirOnly)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion libs/digger_config/yaml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ type TerragruntParsingConfig struct {
CreateProjectName bool `yaml:"createProjectName"`
DefaultTerraformVersion string `yaml:"defaultTerraformVersion"`
DefaultWorkflow string `yaml:"defaultWorkflow"`
FilterPath string `yaml:"filterPath"`
FilterPaths []string `yaml:"filterPath"`
OutputPath string `yaml:"outputPath"`
PreserveWorkflows *bool `yaml:"preserveWorkflows,omitempty"`
PreserveProjects bool `yaml:"preserveProjects"`
Expand Down

0 comments on commit 1ca2067

Please sign in to comment.