Fixing registry host discovery (Terragrunt Provider Cache) (#3432)

* fix: host discovery

* fix: tests

* fix: grammar

* chore: code improvements

* fix: strict-lint

.strict.golangci.yml

@@ -10,6 +10,9 @@ output:
   print-linter-name: true
 
 linters-settings:
+  wrapcheck:
+    ignorePackageGlobs:
+      - github.com/gruntwork-io/terragrunt/*
   lll:
     line-length: 140
   staticcheck:

cli/provider_cache.go

@@ -2,11 +2,9 @@ package cli
 
 import (
 	"context"
-	liberrors "errors"
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -16,6 +14,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/gruntwork-io/go-commons/errors"
 	"github.com/gruntwork-io/terragrunt/options"
+	"github.com/gruntwork-io/terragrunt/pkg/cli"
 	"github.com/gruntwork-io/terragrunt/shell"
 	"github.com/gruntwork-io/terragrunt/terraform"
 	"github.com/gruntwork-io/terragrunt/terraform/cache"
@@ -147,23 +146,29 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
 	}, nil
 }
 
-func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *options.TerragruntOptions, args []string) (*util.CmdOutput, error) {
+// TerraformCommandHook warms up the providers cache, creates `.terraform.lock.hcl` and runs the `tofu/terraform init`
+// command with using this cache. Used as a hook function that is called after running the target tofu/terraform command.
+// For example, if the target command is `tofu plan`, it will be intercepted before it is run in the `/shell` package,
+// then control will be passed to this function to init the working directory using cached providers.
+func (cache *ProviderCache) TerraformCommandHook(
+	ctx context.Context,
+	opts *options.TerragruntOptions,
+	args cli.Args,
+) (*util.CmdOutput, error) {
 	// To prevent a loop
 	ctx = shell.ContextWithTerraformCommandHook(ctx, nil)
 
 	var (
 		cliConfigFilename    = filepath.Join(opts.WorkingDir, localCLIFilename)
-		cacheRequestID       = uuid.New().String()
-		envs                 = providerCacheEnvironment(opts, cliConfigFilename)
-		commandsArgs         = convertToMultipleCommandsByPlatforms(args)
+		env                  = providerCacheEnvironment(opts, cliConfigFilename)
 		skipRunTargetCommand bool
 	)
 
 	// Use Hook only for the `terraform init` command, which can be run explicitly by the user or Terragrunt's `auto-init` feature.
 	switch {
-	case util.FirstArg(args) == terraform.CommandNameInit:
+	case args.CommandName() == terraform.CommandNameInit:
 		// Provider caching for `terraform init` command.
-	case util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock:
+	case args.CommandName() == terraform.CommandNameProviders && args.SubCommandName() == terraform.CommandNameLock:
 		// Provider caching for `terraform providers lock` command.
 		// Since the Terragrunt provider cache server creates the cache and generates the lock file, we don't need to run the `terraform providers lock ...` command at all.
 		skipRunTargetCommand = true
@@ -172,6 +177,29 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
 		return shell.RunTerraformCommandWithOutput(ctx, opts, args...)
 	}
 
+	if output, err := cache.warmUpCache(ctx, opts, cliConfigFilename, args, env); err != nil {
+		return output, err
+	}
+
+	if skipRunTargetCommand {
+		return &util.CmdOutput{}, nil
+	}
+
+	return cache.runTerraformWithCache(ctx, opts, cliConfigFilename, args, env)
+}
+
+func (cache *ProviderCache) warmUpCache(
+	ctx context.Context,
+	opts *options.TerragruntOptions,
+	cliConfigFilename string,
+	args cli.Args,
+	env map[string]string,
+) (*util.CmdOutput, error) {
+	var (
+		cacheRequestID = uuid.New().String()
+		commandsArgs   = convertToMultipleCommandsByPlatforms(args)
+	)
+
 	// Create terraform cli config file that enables provider caching and does not use provider cache dir
 	if err := cache.createLocalCLIConfig(ctx, opts, cliConfigFilename, cacheRequestID); err != nil {
 		return nil, err
@@ -183,7 +211,7 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
 	// It's low cost operation, because it does not cache the same provider twice, but only new previously non-existent providers.
 
 	for _, args := range commandsArgs {
-		if output, err := runTerraformCommand(ctx, opts, args, envs); err != nil {
+		if output, err := runTerraformCommand(ctx, opts, args, env); err != nil {
 			return output, err
 		}
 	}
@@ -193,10 +221,18 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
 		return nil, err
 	}
 
-	if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil {
-		return nil, err
-	}
+	err = getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches)
+
+	return nil, err
+}
 
+func (cache *ProviderCache) runTerraformWithCache(
+	ctx context.Context,
+	opts *options.TerragruntOptions,
+	cliConfigFilename string,
+	args cli.Args,
+	env map[string]string,
+) (*util.CmdOutput, error) {
 	// Create terraform cli config file that uses provider cache dir
 	if err := cache.createLocalCLIConfig(ctx, opts, cliConfigFilename, ""); err != nil {
 		return nil, err
@@ -208,11 +244,7 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
 	}
 
 	cloneOpts.WorkingDir = opts.WorkingDir
-	cloneOpts.Env = envs
-
-	if skipRunTargetCommand {
-		return &util.CmdOutput{}, nil
-	}
+	cloneOpts.Env = env
 
 	return shell.RunTerraformCommandWithOutput(ctx, cloneOpts, args...)
 }
@@ -255,22 +287,15 @@ func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *opti
 	for _, registryName := range opts.ProviderCacheRegistryNames {
 		providerInstallationIncludes = append(providerInstallationIncludes, registryName+"/*/*")
 
-		urls, err := DiscoveryURL(ctx, registryName)
+		apiURLs, err := cache.Server.DiscoveryURL(ctx, registryName)
 		if err != nil {
-			if !liberrors.As(err, &NotFoundWellKnownURL{}) {
-				return err
-			}
-
-			urls = DefaultRegistryURLs
-			opts.Logger.Debugf("Unable to discover %q registry URLs, reason: %q, use default URLs: %s", registryName, err, urls)
-		} else {
-			opts.Logger.Debugf("Discovered %q registry URLs: %s", registryName, urls)
+			return err
 		}
 
 		cfg.AddHost(registryName, map[string]string{
-			"providers.v1": fmt.Sprintf("%s/%s/%s/%s/", cache.ProviderController.URL(), cacheRequestID, url.PathEscape(urls.ProvidersV1), registryName),
+			"providers.v1": fmt.Sprintf("%s/%s/%s/", cache.ProviderController.URL(), cacheRequestID, registryName),
 			// Since Terragrunt Provider Cache only caches providers, we need to route module requests to the original registry.
-			"modules.v1": fmt.Sprintf("https://%s%s", registryName, urls.ModulesV1),
+			"modules.v1": fmt.Sprintf("https://%s%s", registryName, apiURLs.ModulesV1),
 		})
 	}
 

cli/provider_cache_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -50,53 +49,52 @@ func TestProviderCache(t *testing.T) {
 
 	opts := []cache.Option{cache.WithToken(token)}
 
-	registryPrefix := url.PathEscape("/v1/providers/")
-
 	testCases := []struct {
 		opts               []cache.Option
-		urlPath            string
+		fullURLPath        string
+		relURLPath         string
 		expectedStatusCode int
 		expectedBodyReg    *regexp.Regexp
 		expectedCachePath  string
 	}{
 		{
 			opts:               opts,
-			urlPath:            "/.well-known/terraform.json",
+			fullURLPath:        "/.well-known/terraform.json",
 			expectedStatusCode: http.StatusOK,
 			expectedBodyReg:    regexp.MustCompile(regexp.QuoteMeta(`{"providers.v1":"/v1/providers"}`)),
 		},
 		{
 			opts:               append(opts, cache.WithToken("")),
-			urlPath:            "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/aws/versions",
+			relURLPath:         "/cache/registry.terraform.io/hashicorp/aws/versions",
 			expectedStatusCode: http.StatusUnauthorized,
 		},
 		{
 			opts:               opts,
-			urlPath:            "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/aws/versions",
+			relURLPath:         "/cache/registry.terraform.io/hashicorp/aws/versions",
 			expectedStatusCode: http.StatusOK,
 			expectedBodyReg:    regexp.MustCompile(regexp.QuoteMeta(`"version":"5.36.0","protocols":["5.0"],"platforms"`)),
 		},
 		{
 			opts:               opts,
-			urlPath:            "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
+			relURLPath:         "/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
 			expectedStatusCode: http.StatusLocked,
 			expectedCachePath:  "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
 		},
 		{
 			opts:               opts,
-			urlPath:            "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/template/2.2.0/download/linux/amd64",
+			relURLPath:         "/cache/registry.terraform.io/hashicorp/template/2.2.0/download/linux/amd64",
 			expectedStatusCode: http.StatusLocked,
 			expectedCachePath:  "registry.terraform.io/hashicorp/template/2.2.0/linux_amd64/terraform-provider-template_v2.2.0_x4",
 		},
 		{
 			opts:               opts,
-			urlPath:            fmt.Sprintf("/v1/providers/cache/%s/registry.terraform.io/hashicorp/template/1234.5678.9/download/%s/%s", registryPrefix, runtime.GOOS, runtime.GOARCH),
+			relURLPath:         fmt.Sprintf("/cache/registry.terraform.io/hashicorp/template/1234.5678.9/download/%s/%s", runtime.GOOS, runtime.GOARCH),
 			expectedStatusCode: http.StatusLocked,
 			expectedCachePath:  createFakeProvider(t, pluginCacheDir, fmt.Sprintf("registry.terraform.io/hashicorp/template/1234.5678.9/%s_%s/terraform-provider-template_1234.5678.9_x5", runtime.GOOS, runtime.GOARCH)),
 		},
 		{
 			opts:               opts,
-			urlPath:            "/v1/providers//" + registryPrefix + "/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
+			relURLPath:         "//registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
 			expectedStatusCode: http.StatusOK,
 			expectedBodyReg:    regexp.MustCompile(`\{.*` + regexp.QuoteMeta(`"download_url":"http://127.0.0.1:`) + `\d+` + regexp.QuoteMeta(`/downloads/releases.hashicorp.com/terraform-provider-aws/5.36.0/terraform-provider-aws_5.36.0_darwin_arm64.zip"`) + `.*\}`),
 		},
@@ -129,7 +127,11 @@ func TestProviderCache(t *testing.T) {
 			})
 
 			urlPath := server.ProviderController.URL()
-			urlPath.Path = testCase.urlPath
+			urlPath.Path += testCase.relURLPath
+
+			if testCase.fullURLPath != "" {
+				urlPath.Path = testCase.fullURLPath
+			}
 
 			req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlPath.String(), nil)
 			require.NoError(t, err)

pkg/cli/args.go

@@ -36,13 +36,18 @@ func (args Args) First() string {
 	return args.Get(0)
 }
 
-// Last returns the last argument or a blank string
+// Second returns the first argument or a blank string.
+func (args Args) Second() string {
+	return args.Get(1)
+}
+
+// Last returns the last argument or a blank string.
 func (args Args) Last() string {
 	return args.Get(len(args) - 1)
 }
 
 // Tail returns the rest of the arguments (not the first one)
-// or else an empty string slice
+// or else an empty string slice.
 func (args Args) Tail() Args {
 	if args.Len() < tailMinArgsLen {
 		return []string{}
@@ -101,7 +106,8 @@ func (args Args) Normalize(acts ...NormalizeActsType) Args {
 	return strArgs
 }
 
-// CommandName returns the first value if it starts without a dash `-`, otherwise that means the args do not consist any command and an empty string is returned.
+// CommandName returns the first value if it starts without a dash `-`,
+// otherwise that means the args do not consist any command and an empty string is returned.
 func (args Args) CommandName() string {
 	name := args.First()
 
@@ -112,6 +118,18 @@ func (args Args) CommandName() string {
 	return ""
 }
 
+// SubCommandName returns the second value if it starts without a dash `-`,
+// otherwise that means the args do not consist a subcommand and an empty string is returned.
+func (args Args) SubCommandName() string {
+	name := args.Second()
+
+	if !strings.HasPrefix(name, "-") {
+		return name
+	}
+
+	return ""
+}
+
 // Contains returns true if args contains the given `target` arg.
 func (args Args) Contains(target string) bool {
 	for _, arg := range args {

shell/context.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/gruntwork-io/terragrunt/internal/cache"
+	"github.com/gruntwork-io/terragrunt/pkg/cli"
 	"github.com/gruntwork-io/terragrunt/util"
 
 	"github.com/gruntwork-io/terragrunt/options"
@@ -18,13 +19,15 @@ const (
 
 type ctxKey byte
 
-type RunShellCommandFunc func(ctx context.Context, opts *options.TerragruntOptions, args []string) (*util.CmdOutput, error)
+// RunShellCommandFunc is a context value for `TerraformCommandContextKey` key, used to intercept shell commands.
+type RunShellCommandFunc func(ctx context.Context, opts *options.TerragruntOptions, args cli.Args) (*util.CmdOutput, error)
 
 func ContextWithTerraformCommandHook(ctx context.Context, fn RunShellCommandFunc) context.Context {
 	ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName))
 	return context.WithValue(ctx, TerraformCommandContextKey, fn)
 }
 
+// TerraformCommandHookFromContext returns `RunShellCommandFunc` from the context if it has been set, otherwise returns nil.
 func TerraformCommandHookFromContext(ctx context.Context) RunShellCommandFunc {
 	if val := ctx.Value(TerraformCommandContextKey); val != nil {
 		if val, ok := val.(RunShellCommandFunc); ok {

terraform/cache/config.go

@@ -72,7 +72,7 @@ type Config struct {
 	shutdownTimeout time.Duration
 
 	services         []services.Service
-	providerHandlers []handlers.ProviderHandler
+	providerHandlers handlers.ProviderHandlers
 
 	logger log.Logger
 }