diff --git a/config/catalog.go b/config/catalog.go index 4516e2c5b..0e14bc475 100644 --- a/config/catalog.go +++ b/config/catalog.go @@ -44,8 +44,8 @@ func (cfg *CatalogConfig) String() string { return fmt.Sprintf("Catalog{URLs = %v}", cfg.URLs) } -func (cfg *CatalogConfig) normalize(cofnigPath string) { - configDir := filepath.Dir(cofnigPath) +func (cfg *CatalogConfig) normalize(configPath string) { + configDir := filepath.Dir(configPath) // transform relative paths to absolute ones for i, url := range cfg.URLs { diff --git a/docs/_docs/02_features/engine.md b/docs/_docs/02_features/engine.md index 0e2132fa8..752b6efe6 100644 --- a/docs/_docs/02_features/engine.md +++ b/docs/_docs/02_features/engine.md @@ -19,7 +19,7 @@ To try it out, all you need to do is include the following in your `terragrunt.h ```hcl engine { source = "github.com/gruntwork-io/terragrunt-engine-opentofu" - version = "v0.0.5" + version = "v0.0.7" } ``` @@ -27,7 +27,13 @@ This example leverages the official OpenTofu engine, [publicly available on GitH This engine currently leverages the locally available installation of the `tofu` binary, just like Terragrunt does by default without use of engine configurations. It provides a convenient example of how to build engines for Terragrunt. -In the future, this plugin will expand in capability to include additional features and configurations. +In the future, this engine will expand in capability to include additional features and configurations. + +Due to the fact that this functionality is still experimental, and not recommended for general production usage, set the following environment variable to opt-in to this functionality: + +```sh +export TG_EXPERIMENTAL_ENGINE=1 +``` ### Use Cases @@ -45,7 +51,7 @@ e.g. ### HTTPS Sources -Use an HTTP(S) URL to specify the path to the plugin: +Use an HTTP(S) URL to specify the path to the engine: ```hcl engine { @@ -67,9 +73,9 @@ engine { ### Parameters * `source`: (Required) The source of the plugin. Multiple engine approaches are supported, including GitHub repositories, HTTP(S) paths, and local absolute paths. -* `version`: (Required for GitHub) The version of the plugin to download from GitHub releases. +* `version`: The version of the engine to download from GitHub releases, if not specified, the latest release is always downloaded. * `type`: (Optional) Currently, the only supported type is `rpc`. -* `meta`: (Optional) A block for setting plugin-specific metadata. This can include various configuration settings required by the plugin. +* `meta`: (Optional) A block for setting engine-specific metadata. This can include various configuration settings required by the engine. ### Caching @@ -91,22 +97,16 @@ To disable this feature, set the environment variable: export TG_ENGINE_SKIP_CHECK=0 ``` -Due to the fact that this functionality is still experimental, and not recommended for general production usage, set the following environment variable to opt-in to this functionality: - -```sh -export TG_EXPERIMENTAL_ENGINE=1 -``` - ### Engine Metadata The `meta` block is used to pass metadata to the engine. This metadata can be used to configure the engine or pass additional information to the engine. -The metadata block is a map of key-value pairs. Plugins can read the information passed via the metadata map to configure themselves or to pass additional information to the engine. +The metadata block is a map of key-value pairs. Engines can read the information passed via the metadata map to configure themselves. ```hcl engine { - source = "/home/users/iac-engines/my-custom-plugin" - # Optionally set metadata for the plugin. + source = "/home/users/iac-engines/my-custom-engine" + # Optionally set metadata for the engine. meta = { key_1 = ["value1", "value2"] key_2 = "1.6.0" diff --git a/engine/engine.go b/engine/engine.go index 445450da4..9c1de0c81 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -8,6 +8,7 @@ import ( goErrors "errors" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -16,6 +17,8 @@ import ( "strings" "sync" + "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/hashicorp/go-getter" "github.com/mholt/archiver/v3" @@ -46,8 +49,10 @@ const ( ChecksumFileNameFormat = "terragrunt-iac-%s_%s_%s_SHA256SUMS" EngineCachePathEnv = "TG_ENGINE_CACHE_PATH" EngineSkipCheckEnv = "TG_ENGINE_SKIP_CHECK" + defaultEngineRepoRoot = "github.com/" TerraformCommandContextKey engineClientsKey = iota LocksContextKey engineLocksKey = iota + LatestVersionsContextKey engineLocksKey = iota ) type engineClientsKey byte @@ -130,6 +135,7 @@ func WithEngineValues(ctx context.Context) context.Context { ctx = context.WithValue(ctx, TerraformCommandContextKey, &sync.Map{}) ctx = context.WithValue(ctx, LocksContextKey, util.NewKeyLocks()) + ctx = context.WithValue(ctx, LatestVersionsContextKey, cache.NewCache[string]("engineVersions")) return ctx } @@ -147,6 +153,18 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error return nil } + // identify engine version if not specified + if len(e.Version) == 0 { + if !strings.Contains(e.Source, "://") { + tag, err := lastReleaseVersion(ctx, opts) + if err != nil { + return errors.WithStackTrace(err) + } + + e.Version = tag + } + } + path, err := engineDir(e) if err != nil { return errors.WithStackTrace(err) @@ -226,6 +244,53 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error return nil } +func lastReleaseVersion(ctx context.Context, opts *options.TerragruntOptions) (string, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", strings.TrimPrefix(opts.Engine.Source, defaultEngineRepoRoot)) + + versionCache, err := engineVersionsCacheFromContext(ctx) + + if err != nil { + return "", errors.WithStackTrace(err) + } + + if val, found := versionCache.Get(ctx, url); found { + return val, nil + } + + type release struct { + Tag string `json:"tag_name"` + } + // query tag from https://api.github.com/repos/{owner}/{repo}/releases/latest + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + + if err != nil { + return "", errors.WithStackTrace(err) + } + + client := &http.Client{} + resp, err := client.Do(req) + + if err != nil { + return "", errors.WithStackTrace(err) + } + + defer resp.Body.Close() //nolint:errcheck + body, err := io.ReadAll(resp.Body) + + if err != nil { + return "", errors.WithStackTrace(err) + } + + var r release + if err := json.Unmarshal(body, &r); err != nil { + return "", errors.WithStackTrace(err) + } + + versionCache.Put(ctx, url, r.Tag) + + return r.Tag, nil +} + func extractArchive(opts *options.TerragruntOptions, downloadFile string, engineFile string) error { if !isArchiveByHeader(downloadFile) { opts.Logger.Info("Downloaded file is not an archive, no extraction needed") @@ -383,6 +448,20 @@ func downloadLocksFromContext(ctx context.Context) (*util.KeyLocks, error) { return result, nil } +func engineVersionsCacheFromContext(ctx context.Context) (*cache.Cache[string], error) { + val := ctx.Value(LatestVersionsContextKey) + if val == nil { + return nil, errors.WithStackTrace(goErrors.New("failed to fetch engine versions cache from context")) + } + + result, ok := val.(*cache.Cache[string]) + if !ok { + return nil, errors.WithStackTrace(goErrors.New("failed to cast engine versions cache from context")) + } + + return result, nil +} + // IsEngineEnabled returns true if the experimental engine is enabled. func IsEngineEnabled() bool { ok, _ := strconv.ParseBool(os.Getenv(EnableExperimentalEngineEnvName)) //nolint:errcheck diff --git a/test/fixtures/engine/opentofu-latest-run-all/app1/main.tf b/test/fixtures/engine/opentofu-latest-run-all/app1/main.tf new file mode 100644 index 000000000..2d915f6fe --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app1/main.tf @@ -0,0 +1,5 @@ + +resource "local_file" "test" { + content = "app1" + filename = "${path.module}/test.txt" +} diff --git a/test/fixtures/engine/opentofu-latest-run-all/app1/terragrunt.hcl b/test/fixtures/engine/opentofu-latest-run-all/app1/terragrunt.hcl new file mode 100644 index 000000000..ca7a37984 --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app1/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = find_in_parent_folders() +} \ No newline at end of file diff --git a/test/fixtures/engine/opentofu-latest-run-all/app2/main.tf b/test/fixtures/engine/opentofu-latest-run-all/app2/main.tf new file mode 100644 index 000000000..2d915f6fe --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app2/main.tf @@ -0,0 +1,5 @@ + +resource "local_file" "test" { + content = "app1" + filename = "${path.module}/test.txt" +} diff --git a/test/fixtures/engine/opentofu-latest-run-all/app2/terragrunt.hcl b/test/fixtures/engine/opentofu-latest-run-all/app2/terragrunt.hcl new file mode 100644 index 000000000..ca7a37984 --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app2/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = find_in_parent_folders() +} \ No newline at end of file diff --git a/test/fixtures/engine/opentofu-latest-run-all/app3/main.tf b/test/fixtures/engine/opentofu-latest-run-all/app3/main.tf new file mode 100644 index 000000000..2d915f6fe --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app3/main.tf @@ -0,0 +1,5 @@ + +resource "local_file" "test" { + content = "app1" + filename = "${path.module}/test.txt" +} diff --git a/test/fixtures/engine/opentofu-latest-run-all/app3/terragrunt.hcl b/test/fixtures/engine/opentofu-latest-run-all/app3/terragrunt.hcl new file mode 100644 index 000000000..ca7a37984 --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app3/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = find_in_parent_folders() +} \ No newline at end of file diff --git a/test/fixtures/engine/opentofu-latest-run-all/app4/main.tf b/test/fixtures/engine/opentofu-latest-run-all/app4/main.tf new file mode 100644 index 000000000..2d915f6fe --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app4/main.tf @@ -0,0 +1,5 @@ + +resource "local_file" "test" { + content = "app1" + filename = "${path.module}/test.txt" +} diff --git a/test/fixtures/engine/opentofu-latest-run-all/app4/terragrunt.hcl b/test/fixtures/engine/opentofu-latest-run-all/app4/terragrunt.hcl new file mode 100644 index 000000000..ca7a37984 --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app4/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = find_in_parent_folders() +} \ No newline at end of file diff --git a/test/fixtures/engine/opentofu-latest-run-all/app5/main.tf b/test/fixtures/engine/opentofu-latest-run-all/app5/main.tf new file mode 100644 index 000000000..2d915f6fe --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app5/main.tf @@ -0,0 +1,5 @@ + +resource "local_file" "test" { + content = "app1" + filename = "${path.module}/test.txt" +} diff --git a/test/fixtures/engine/opentofu-latest-run-all/app5/terragrunt.hcl b/test/fixtures/engine/opentofu-latest-run-all/app5/terragrunt.hcl new file mode 100644 index 000000000..ca7a37984 --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/app5/terragrunt.hcl @@ -0,0 +1,3 @@ +include { + path = find_in_parent_folders() +} \ No newline at end of file diff --git a/test/fixtures/engine/opentofu-latest-run-all/terragrunt.hcl b/test/fixtures/engine/opentofu-latest-run-all/terragrunt.hcl new file mode 100644 index 000000000..e49070c18 --- /dev/null +++ b/test/fixtures/engine/opentofu-latest-run-all/terragrunt.hcl @@ -0,0 +1,4 @@ +engine { + // use latest OpenTofu engine to do basic validation of implementation + source = "github.com/gruntwork-io/terragrunt-engine-opentofu" +} diff --git a/test/integration_engine_test.go b/test/integration_engine_test.go index e6a2a4b39..016d4cea9 100644 --- a/test/integration_engine_test.go +++ b/test/integration_engine_test.go @@ -20,10 +20,11 @@ import ( ) const ( - testFixtureLocalEngine = "fixtures/engine/local-engine" - testFixtureRemoteEngine = "fixtures/engine/remote-engine" - testFixtureOpenTofuEngine = "fixtures/engine/opentofu-engine" - testFixtureOpenTofuRunAll = "fixtures/engine/opentofu-run-all" + testFixtureLocalEngine = "fixtures/engine/local-engine" + testFixtureRemoteEngine = "fixtures/engine/remote-engine" + testFixtureOpenTofuEngine = "fixtures/engine/opentofu-engine" + testFixtureOpenTofuRunAll = "fixtures/engine/opentofu-run-all" + testFixtureOpenTofuLatestRunAll = "fixtures/engine/opentofu-latest-run-all" envVarExperimental = "TG_EXPERIMENTAL_ENGINE" ) @@ -200,6 +201,22 @@ func TestEngineDisableChecksumCheck(t *testing.T) { require.NoError(t, err) } +func TestEngineOpentofuLatestRunAll(t *testing.T) { + t.Setenv(envVarExperimental, "1") + + cleanupTerraformFolder(t, testFixtureOpenTofuLatestRunAll) + tmpEnvPath := copyEnvironment(t, testFixtureOpenTofuLatestRunAll) + rootPath := util.JoinPath(tmpEnvPath, testFixtureOpenTofuLatestRunAll) + + stdout, _, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath)) + require.NoError(t, err) + + assert.Contains(t, stdout, "resource \"local_file\" \"test\"") + assert.Contains(t, stdout, "filename = \"./test.txt\"\n") + assert.Contains(t, stdout, "Tofu Shutdown completed") + assert.Contains(t, stdout, "Apply complete!") +} + func setupEngineCache(t *testing.T) (string, string) { // create temporary folder cacheDir := t.TempDir()