Add support for generating go module deps

docs/examples/go-md2man-1.yml

@@ -1,6 +1,7 @@
 # syntax=ghcr.io/azure/dalec/frontend:latest
 name: go-md2man
 version: 2.0.3
+revision: "1"
 packager: Dalec Example
 vendor: Dalec Example
 license: MIT

docs/examples/go-md2man-2.yml

@@ -1,6 +1,7 @@
 # syntax=ghcr.io/azure/dalec/frontend:latest
 name: go-md2man
 version: 2.0.3
+revision: "1"
 packager: Dalec Example
 vendor: Dalec Example
 license: MIT
@@ -9,27 +10,11 @@ website: https://github.com/cpuguy83/go-md2man
 
 sources:
   src:
+    generate:
+      - gomod: {}
     git:
       url: https://github.com/cpuguy83/go-md2man.git
       commit: "v2.0.3"
-  gomods: # This is required when the build environment does not allow network access. This downloads all the go modules.
-    path: /build/gomodcache # This is the path we will be extracing after running the command below.
-    image:
-      ref: mcr.microsoft.com/oss/go/microsoft/golang:1.21
-      cmd:
-        dir: /build/src
-        mounts:
-          # Mount a source (inline, under `spec`), so our command has access to it.
-          - dest: /build/src
-            spec:
-              git:
-                url: https://github.com/cpuguy83/go-md2man.git
-                commit: "v2.0.3"
-        steps:
-          - command: go mod download
-            env:
-              # This variable controls where the go modules are downloaded to.
-              GOMODCACHE: /build/gomodcache
 
 dependencies:
   build:
@@ -40,7 +25,6 @@ build:
     CGO_ENABLED: "0"
   steps:
     - command: |
-        export GOMODCACHE="$(pwd)/gomods"
         cd src
         go build -o go-md2man .
 

docs/spec.schema.json

@@ -291,6 +291,12 @@
 			],
 			"description": "Frontend encapsulates the configuration for a frontend to forward a build target to."
 		},
+		"GeneratorGomod": {
+			"properties": {},
+			"additionalProperties": false,
+			"type": "object",
+			"description": "GeneratorGomod is used to generate a go module cache from go module sources"
+		},
 		"ImageConfig": {
 			"properties": {
 				"entrypoint": {
@@ -466,6 +472,13 @@
 					},
 					"type": "array",
 					"description": "Excludes is a list of paths underneath `Path` to exclude, everything else is included"
+				},
+				"generate": {
+					"items": {
+						"$ref": "#/$defs/SourceGenerator"
+					},
+					"type": "array",
+					"description": "Generate is the list generators to run on the source.\n\nGenerators are used to generate additional sources from this source.\nAs an example the `godmod` generator can be used to generate a go module cache from a go source.\nHow a genator operates is dependent on the actual generator.\nGeneators may also cauuse modifications to the build environment.\n\nCurrently only one generator is supported: \"gomod\""
 				}
 			},
 			"additionalProperties": false,
@@ -524,6 +537,24 @@
 				"ref"
 			]
 		},
+		"SourceGenerator": {
+			"properties": {
+				"subpath": {
+					"type": "string",
+					"description": "Subpath is the path inside a source to run the generator from."
+				},
+				"gomod": {
+					"$ref": "#/$defs/GeneratorGomod",
+					"description": "Gomod is the go module generator."
+				}
+			},
+			"additionalProperties": false,
+			"type": "object",
+			"required": [
+				"gomod"
+			],
+			"description": "SourceGenerator holds the configuration for a source generator."
+		},
 		"SourceGit": {
 			"properties": {
 				"url": {

frontend/debug/handle_gomod.go

@@ -0,0 +1,60 @@
+package debug
+
+import (
+	"context"
+
+	"github.com/Azure/dalec"
+	"github.com/Azure/dalec/frontend"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/gateway/client"
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+const keyGomodWorker = "context:gomod-worker"
+
+// Gomods outputs all the gomodule dependencies for the spec
+func Gomods(ctx context.Context, client gwclient.Client) (*client.Result, error) {
+	return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
+		sOpt, err := frontend.SourceOptFromClient(ctx, client)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		inputs, err := client.Inputs(ctx)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// Allow the client to override the worker image
+		// This is useful for keeping pre-built worker image, especially for CI.
+		worker, ok := inputs[keyGomodWorker]
+		if !ok {
+			worker = llb.Image("alpine:latest", llb.WithMetaResolver(client)).
+				Run(llb.Shlex("apk add --no-cache go git ca-certificates patch")).Root()
+		}
+
+		st, err := spec.GomodDeps(sOpt, worker)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		def, err := st.Marshal(ctx)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		res, err := client.Solve(ctx, gwclient.SolveRequest{
+			Definition: def.ToPB(),
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+
+		ref, err := res.SingleRef()
+		if err != nil {
+			return nil, nil, err
+		}
+		return ref, nil, nil
+	})
+}

frontend/debug/handler.go

@@ -21,6 +21,10 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 		Name:        "sources",
 		Description: "Outputs all sources from a dalec spec file.",
 	})
+	r.Add("gomods", Gomods, &targets.Target{
+		Name:        "gomods",
+		Description: "Outputs all the gomodule dependencies for the spec",
+	})
 
 	return r.Handle(ctx, client)
 }

frontend/mariner2/handle_rpm.go

@@ -76,7 +76,7 @@ func getWorkerImage(resolver llb.ImageMetaResolver, opts ...llb.ConstraintsOpt)
 	opts = append(opts, dalec.ProgressGroup("Prepare worker image"))
 	return llb.Image(marinerRef, llb.WithMetaResolver(resolver), dalec.WithConstraints(opts...)).
 		Run(
-			shArgs("tdnf install -y rpm-build mariner-rpm-macros build-essential"),
+			shArgs("tdnf install -y rpm-build mariner-rpm-macros build-essential ca-certificates"),
 			defaultTdnfCacheMount(),
 			dalec.WithConstraints(opts...),
 		).
@@ -89,8 +89,8 @@ func installBuildDeps(spec *dalec.Spec, targetKey string, opts ...llb.Constraint
 		if len(deps) == 0 {
 			return in
 		}
-		opts = append(opts, dalec.ProgressGroup("Install build deps"))
 
+		opts = append(opts, dalec.ProgressGroup("Install build deps"))
 		return in.
 			Run(
 				shArgs(fmt.Sprintf("tdnf install --releasever=2.0 -y %s", strings.Join(deps, " "))),
@@ -102,12 +102,16 @@ func installBuildDeps(spec *dalec.Spec, targetKey string, opts ...llb.Constraint
 }
 
 func specToRpmLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
-	br, err := rpm.SpecToBuildrootLLB(spec, sOpt, targetKey, opts...)
+	base, err := getSpecWorker(sOpt.Resolver, spec, targetKey, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
+	br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}
 	specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")
 
-	base := getWorkerImage(sOpt.Resolver, opts...).With(installBuildDeps(spec, targetKey, opts...))
-	return rpm.Build(br, base, specPath, opts...), nil
+	return rpm.Build(br, base.With(installBuildDeps(spec, targetKey, opts...)), specPath, opts...), nil
 }

frontend/mariner2/handler.go

@@ -2,9 +2,13 @@ package mariner2
 
 import (
 	"context"
+	"errors"
+	"slices"
 
+	"github.com/Azure/dalec"
 	"github.com/Azure/dalec/frontend"
 	"github.com/Azure/dalec/frontend/rpm"
+	"github.com/moby/buildkit/client/llb"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
 	bktargets "github.com/moby/buildkit/frontend/subrequests/targets"
 )
@@ -20,7 +24,8 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 		Name:        "rpm",
 		Description: "Builds an rpm and src.rpm for mariner2.",
 	})
-	mux.Add("rpm/debug", rpm.HandleDebug(), nil)
+
+	mux.Add("rpm/debug", handleDebug, nil)
 
 	mux.Add("container", handleContainer, &bktargets.Target{
 		Name:        "container",
@@ -35,3 +40,23 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 
 	return mux.Handle(ctx, client)
 }
+
+func handleDebug(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
+	return rpm.HandleDebug(getSpecWorker)(ctx, client)
+}
+
+func getSpecWorker(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	st := getWorkerImage(resolver, opts...)
+	if spec.HasGomods() {
+		deps := spec.GetBuildDeps(targetKey)
+		hasGolang := func(s string) bool {
+			return s == "golang" || s == "msft-golang"
+		}
+
+		if !slices.ContainsFunc(deps, hasGolang) {
+			return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps")
+		}
+		st = st.With(installBuildDeps(spec, targetKey, opts...))
+	}
+	return st, nil
+}

frontend/rpm/handle_buildroot.go

@@ -11,15 +11,22 @@ import (
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-func HandleBuildroot() gwclient.BuildFunc {
+type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error)
+
+func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc {
 	return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
 		return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
 			sOpt, err := frontend.SourceOptFromClient(ctx, client)
 			if err != nil {
 				return nil, nil, err
 			}
 
-			st, err := SpecToBuildrootLLB(spec, sOpt, targetKey)
+			worker, err := wf(sOpt.Resolver, spec, targetKey)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			st, err := SpecToBuildrootLLB(worker, spec, sOpt, targetKey)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -47,13 +54,13 @@ func HandleBuildroot() gwclient.BuildFunc {
 }
 
 // SpecToBuildrootLLB converts a dalec.Spec to an rpm buildroot
-func SpecToBuildrootLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+func SpecToBuildrootLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
 	if err := ValidateSpec(spec); err != nil {
 		return llb.Scratch(), fmt.Errorf("invalid spec: %w", err)
 	}
 	opts = append(opts, dalec.ProgressGroup("Create RPM buildroot"))
 
-	sources, err := Dalec2SourcesLLB(spec, sOpt, opts...)
+	sources, err := Dalec2SourcesLLB(worker, spec, sOpt, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}

frontend/rpm/handle_sources.go

@@ -10,6 +10,7 @@ import (
 	"github.com/moby/buildkit/client/llb"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 )
 
 // TarImageRef is the image used to create tarballs of sources
@@ -41,15 +42,20 @@ func tar(src llb.State, dest string, opts ...llb.ConstraintsOpt) llb.State {
 	return worker.AddMount(outBase, llb.Scratch())
 }
 
-func HandleSources() gwclient.BuildFunc {
+func HandleSources(wf WorkerFunc) gwclient.BuildFunc {
 	return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
 		return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
 			sOpt, err := frontend.SourceOptFromClient(ctx, client)
 			if err != nil {
 				return nil, nil, err
 			}
 
-			sources, err := Dalec2SourcesLLB(spec, sOpt)
+			worker, err := wf(sOpt.Resolver, spec, targetKey)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			sources, err := Dalec2SourcesLLB(worker, spec, sOpt)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -78,36 +84,19 @@ func HandleSources() gwclient.BuildFunc {
 	}
 }
 
-func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.State, error) {
+func Dalec2SourcesLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.State, error) {
 	// Sort the map keys so that the order is consistent This shouldn't be
 	// needed when MergeOp is supported, but when it is not this will improve
 	// cache hits for callers of this function.
 	sorted := dalec.SortMapKeys(spec.Sources)
 
 	out := make([]llb.State, 0, len(spec.Sources))
+
 	for _, k := range sorted {
 		src := spec.Sources[k]
 		isDir := dalec.SourceIsDir(src)
 
-		s := ""
-		switch {
-		case src.DockerImage != nil:
-			s = src.DockerImage.Ref
-		case src.Git != nil:
-			s = src.Git.URL
-		case src.HTTP != nil:
-			s = src.HTTP.URL
-		case src.Context != nil:
-			s = src.Context.Name
-		case src.Build != nil:
-			s = fmt.Sprintf("%v", src.Build.Source)
-		case src.Inline != nil:
-			s = "inline"
-		default:
-			return nil, fmt.Errorf("no non-nil source provided")
-		}
-
-		pg := dalec.ProgressGroup("Add spec source: " + k + " " + s)
+		pg := dalec.ProgressGroup("Add spec source: " + k)
 		st, err := src.AsState(k, sOpt, append(opts, pg)...)
 		if err != nil {
 			return nil, err
@@ -120,5 +109,15 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.Const
 		}
 	}
 
+	opts = append(opts, dalec.ProgressGroup("Add gomod sources"))
+	st, err := spec.GomodDeps(sOpt, worker, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "error adding gomod sources")
+	}
+
+	if st != nil {
+		out = append(out, tar(*st, gomodsName+".tar.gz", opts...))
+	}
+
 	return out, nil
 }