feat: Add gomod git auth

docs/spec.schema.json

@@ -430,6 +430,13 @@
 					},
 					"type": "array",
 					"description": "Paths is the list of paths to run the generator on. Used to generate multi-module in a single source."
+				},
+				"auth": {
+					"additionalProperties": {
+						"$ref": "#/$defs/GitAuthWithUsername"
+					},
+					"type": "object",
+					"description": "Auth is the git authorization to use for gomods. The keys are the hosts, and the values are the auth to use for that host."
 				}
 			},
 			"additionalProperties": false,
@@ -454,6 +461,28 @@
 			"additionalProperties": false,
 			"type": "object"
 		},
+		"GitAuthWithUsername": {
+			"properties": {
+				"header": {
+					"type": "string",
+					"description": "Header is the name of the secret which contains the git auth header.\nwhen using git auth header based authentication.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+				},
+				"token": {
+					"type": "string",
+					"description": "Token is the name of the secret which contains a git auth token when using\ntoken based authentication.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+				},
+				"ssh": {
+					"type": "string",
+					"description": "SSH is the name of the secret which contains the ssh auth into when using\nssh based auth.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+				},
+				"ssh_username": {
+					"type": "string",
+					"description": "SSHUsername is the SSHUsername to use when connecting to a git repo"
+				}
+			},
+			"additionalProperties": false,
+			"type": "object"
+		},
 		"ImageConfig": {
 			"properties": {
 				"entrypoint": {

frontend/debug/handle_gomod.go

@@ -31,7 +31,7 @@ func Gomods(ctx context.Context, client gwclient.Client) (*client.Result, error)
 		worker, ok := inputs[keyGomodWorker]
 		if !ok {
 			worker = llb.Image("alpine:latest", llb.WithMetaResolver(client)).
-				Run(llb.Shlex("apk add --no-cache go git ca-certificates patch")).Root()
+				Run(llb.Shlex("apk add --no-cache go git ca-certificates patch openssh")).Root()
 		}
 
 		st, err := spec.GomodDeps(sOpt, worker)

generator_gomod.go

@@ -1,14 +1,20 @@
 package dalec
 
 import (
+	"bytes"
+	_ "embed"
+	"fmt"
 	"path/filepath"
+	"sort"
 
 	"github.com/moby/buildkit/client/llb"
 	"github.com/pkg/errors"
 )
 
 const (
-	gomodCacheDir = "/go/pkg/mod"
+	gomodCacheDir       = "/go/pkg/mod"
+	gitConfigMountpoint = "/dev/shm/git"
+	scriptRelativePath  = "/go_mod_download.sh"
 )
 
 func (s *Source) isGomod() bool {
@@ -32,7 +38,12 @@ func (s *Spec) HasGomods() bool {
 
 func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.ConstraintsOpt) func(llb.State) llb.State {
 	return func(in llb.State) llb.State {
-		workDir := "/work/src"
+		const (
+			fourKB           = 4096
+			workDir          = "/work/src"
+			scriptMountPoint = "/tmp/mnt"
+		)
+
 		joinedWorkDir := filepath.Join(workDir, g.Subpath)
 		srcMount := llb.AddMount(workDir, srcSt)
 
@@ -41,10 +52,17 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.Constrai
 			paths = []string{"."}
 		}
 
+		sort.Strings(paths)
+		script := g.gitconfigGeneratorScript()
+		scriptPath := filepath.Join(scriptMountPoint, scriptRelativePath)
+
 		for _, path := range paths {
 			in = worker.Run(
-				ShArgs("go mod download"),
+				ShArgs(scriptPath),
 				llb.AddEnv("GOPATH", "/go"),
+				g.withGomodSecretsAndSockets(),
+				llb.AddMount(scriptMountPoint, script),
+				llb.AddMount(gitConfigMountpoint, llb.Scratch(), llb.Tmpfs(llb.TmpfsSize(fourKB))), // to house the gitconfig, which has secrets
 				llb.Dir(filepath.Join(joinedWorkDir, path)),
 				srcMount,
 				WithConstraints(opts...),
@@ -54,6 +72,93 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.Constrai
 	}
 }
 
+func (g *SourceGenerator) gitconfigGeneratorScript() llb.State {
+	var (
+		script bytes.Buffer
+		noop   = func() {}
+
+		createPreamble = func() {
+			fmt.Fprintln(&script, `set -eu`)
+			fmt.Fprintf(&script, `ln -sf %s/.gitconfig "${HOME}/.gitconfig"`, gitConfigMountpoint)
+			script.WriteRune('\n')
+		}
+	)
+
+	fmt.Fprintln(&script, `#!/usr/bin/env sh`)
+
+	for host, auth := range g.Gomod.Auth {
+		// Only do this the first time through the loop
+		createPreamble()
+		createPreamble = noop
+
+		var headerArg string
+		if auth.Header != "" {
+			headerArg = fmt.Sprintf(`Authorization: ${%s}`, auth.Header)
+		}
+
+		if auth.Token != "" && headerArg == "" {
+			line := fmt.Sprintf(`tkn="$(echo -n "x-access-token:${%s}" | base64)"`, auth.Token)
+			fmt.Fprintln(&script, line)
+
+			headerArg = `Authorization: basic ${tkn}`
+		}
+
+		if headerArg != "" {
+			fmt.Fprintf(&script, `git config --global http."https://%s".extraheader "%s"`, host, headerArg)
+			script.WriteRune('\n')
+			continue
+		}
+
+		if auth.SSH != "" {
+			username := "git"
+			if auth.SSHUsername != "" {
+				username = auth.SSHUsername
+			}
+
+			fmt.Fprintf(&script, `git config --global "url.ssh://%[1]s@%[2]s/.insteadOf" https://%[2]s/`, username, host)
+			script.WriteRune('\n')
+		}
+	}
+
+	fmt.Fprintln(&script, "go mod download")
+	return llb.Scratch().File(llb.Mkfile(scriptRelativePath, 0o755, script.Bytes()))
+}
+
+func (g *SourceGenerator) withGomodSecretsAndSockets() llb.RunOption {
+	envIsSet := false
+	return runOptionFunc(func(ei *llb.ExecInfo) {
+		if g.Gomod == nil {
+			return
+		}
+
+		secrets := make(map[string]struct{}, len(g.Gomod.Auth))
+		for _, auth := range g.Gomod.Auth {
+			if auth.Token != "" {
+				secrets[auth.Token] = struct{}{}
+				continue
+			}
+
+			if auth.Header != "" {
+				secrets[auth.Header] = struct{}{}
+				continue
+			}
+
+			if auth.SSH != "" {
+				llb.AddSSHSocket(llb.SSHID(auth.SSH)).SetRunOption(ei)
+
+				if !envIsSet {
+					llb.AddEnv("GIT_SSH_COMMAND", `ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no`).SetRunOption(ei)
+					envIsSet = true
+				}
+			}
+		}
+
+		for secret := range secrets {
+			secretToEnv(secret).SetRunOption(ei)
+		}
+	})
+}
+
 func (s *Spec) gomodSources() map[string]Source {
 	sources := map[string]Source{}
 	for name, src := range s.Sources {

source.go

@@ -863,3 +863,8 @@ func (s *Source) validate(failContext ...string) (retErr error) {
 
 	return retErr
 }
+
+func secretToEnv(secretName string, opts ...llb.SecretOption) llb.RunOption {
+	allOpts := append([]llb.SecretOption{llb.SecretID(secretName), llb.SecretAsEnv(true)}, opts...)
+	return llb.AddSecret(secretName, allOpts...)
+}