diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index cc6cbe73a..87e89f545 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -52,25 +52,80 @@ func handleRPM(w worker) gwclient.BuildFunc { } } -func installBuildDeps(w worker, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) llb.StateOption { - return func(in llb.State) llb.State { - deps := spec.GetBuildDeps(targetKey) - if len(deps) == 0 { - return in +// Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies +func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc { + // depsOnly is a simple dalec spec that only includes build dependencies and their constraints + depsOnly := dalec.Spec{ + Name: fmt.Sprintf("%s-build-dependencies", packageName), + Description: "Provides build dependencies for mariner2 and azlinux3", + Version: "1.0", + License: "Apache 2.0", + Revision: "1", + Dependencies: &dalec.PackageDependencies{ + Runtime: deps, + }, + } + + return func(ctx context.Context, client gwclient.Client, sOpt dalec.SourceOpts) (llb.RunOption, error) { + pg := dalec.ProgressGroup("Building container for build dependencies") + + // create an RPM with just the build dependencies, using our same base worker + rpmDir, err := specToRpmLLB(ctx, w, client, &depsOnly, sOpt, target, pg) + if err != nil { + return nil, err } + + var opts []llb.ConstraintsOpt opts = append(opts, dalec.ProgressGroup("Install build deps")) - return in.Run(w.Install(deps, installWithConstraints(opts)), dalec.WithConstraints(opts...)).Root() + rpmMountDir := "/tmp/rpms" + + installOpts = append([]installOpt{ + noGPGCheck, + withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), + installWithConstraints(opts), + }, installOpts...) + + // install the built RPMs into the worker itself + return w.Install([]string{"/tmp/rpms/*/*.rpm"}, installOpts...), nil } } +func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { + deps := spec.GetBuildDeps(targetKey) + if len(deps) == 0 { + return func(in llb.State) llb.State { return in }, nil + } + + sOpt, err := frontend.SourceOptFromClient(ctx, client) + if err != nil { + return nil, err + } + + opts = append(opts, dalec.ProgressGroup("Install build deps")) + + installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt) + if err != nil { + return nil, err + } + + return func(in llb.State) llb.State { + return in.Run(installOpt, dalec.WithConstraints(opts...)).Root() + }, nil +} + func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { base, err := w.Base(sOpt, opts...) - base = base.With(installBuildDeps(w, spec, targetKey, opts...)) if err != nil { return llb.Scratch(), err } + installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + if err != nil { + return llb.Scratch(), err + } + base = base.With(installOpt) + br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...) if err != nil { return llb.Scratch(), err diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index 97396c949..dc2498f7e 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -18,6 +18,8 @@ const ( tdnfCacheDir = "/var/cache/tdnf" ) +type installFunc func(context.Context, gwclient.Client, dalec.SourceOpts) (llb.RunOption, error) + type worker interface { Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) Install(pkgs []string, opts ...installOpt) llb.RunOption @@ -60,18 +62,19 @@ func handleDebug(w worker) gwclient.BuildFunc { if err != nil { return nil, err } - return rpm.HandleDebug(getSpecWorker(w, sOpt))(ctx, client) + return rpm.HandleDebug(getSpecWorker(ctx, w, client, sOpt))(ctx, client) } } -func getSpecWorker(w worker, sOpt dalec.SourceOpts) rpm.WorkerFunc { +func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt dalec.SourceOpts) rpm.WorkerFunc { return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { st, err := w.Base(sOpt, opts...) if err != nil { return llb.Scratch(), err } if spec.HasGomods() { - deps := spec.GetBuildDeps(targetKey) + deps := dalec.SortMapKeys(spec.GetBuildDeps(targetKey)) + hasGolang := func(s string) bool { return s == "golang" || s == "msft-golang" } @@ -79,7 +82,13 @@ func getSpecWorker(w worker, sOpt dalec.SourceOpts) rpm.WorkerFunc { if !slices.ContainsFunc(deps, hasGolang) { return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps") } - st = st.With(installBuildDeps(w, spec, targetKey, opts...)) + + installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + if err != nil { + return llb.Scratch(), err + } + + st = st.With(installOpt) } return st, nil } diff --git a/frontend/azlinux/install.go b/frontend/azlinux/install.go index 6595ca246..c933e8335 100644 --- a/frontend/azlinux/install.go +++ b/frontend/azlinux/install.go @@ -20,6 +20,9 @@ type installConfig struct { // this acts like installing to a chroot. root string + // Additional mounts to add to the tdnf install command (useful if installing RPMS which are mounted to a local directory) + mounts []llb.RunOption + constraints []llb.ConstraintsOpt } @@ -29,6 +32,12 @@ func noGPGCheck(cfg *installConfig) { cfg.noGPGCheck = true } +func withMounts(opts ...llb.RunOption) installOpt { + return func(cfg *installConfig) { + cfg.mounts = append(cfg.mounts, opts...) + } +} + func withManifests(cfg *installConfig) { cfg.manifest = true } @@ -104,7 +113,7 @@ const manifestSh = "manifest.sh" func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption { cmdFlags := tdnfInstallFlags(cfg) - cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " ")) + cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " ")) var runOpts []llb.RunOption @@ -118,5 +127,7 @@ func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption } runOpts = append(runOpts, dalec.ShArgs(cmdArgs)) + runOpts = append(runOpts, cfg.mounts...) + return dalec.WithRunOptions(runOpts...) } diff --git a/frontend/windows/handle_zip.go b/frontend/windows/handle_zip.go index 713068840..cc95b05a8 100644 --- a/frontend/windows/handle_zip.go +++ b/frontend/windows/handle_zip.go @@ -128,7 +128,10 @@ func withSourcesMounted(dst string, states map[string]llb.State, sources map[str } func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, client gwclient.Client, sOpt dalec.SourceOpts, targetKey string) (llb.State, error) { - worker = worker.With(installBuildDeps(spec.GetBuildDeps(targetKey))) + deps := dalec.SortMapKeys(spec.GetBuildDeps(targetKey)) + + // note: we do not yet support pinning build dependencies for windows workers + worker = worker.With(installBuildDeps(deps)) sources, err := specToSourcesLLB(worker, spec, sOpt) if err != nil { diff --git a/helpers.go b/helpers.go index 4889b8efe..d3b472896 100644 --- a/helpers.go +++ b/helpers.go @@ -285,7 +285,7 @@ func (s *Spec) GetRuntimeDeps(targetKey string) []string { } -func (s *Spec) GetBuildDeps(targetKey string) []string { +func (s *Spec) GetBuildDeps(targetKey string) map[string]PackageConstraints { var deps *PackageDependencies if t, ok := s.Targets[targetKey]; ok { deps = t.Dependencies @@ -298,7 +298,7 @@ func (s *Spec) GetBuildDeps(targetKey string) []string { } } - return SortMapKeys(deps.Build) + return deps.Build } func (s *Spec) GetTestDeps(targetKey string) []string { diff --git a/test/azlinux_test.go b/test/azlinux_test.go index c7252d490..86180133a 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -111,7 +111,7 @@ type workerConfig struct { type targetConfig struct { // Package is the target for creating a package. Package string - // Container is the target for creating a container. + // Container is the target for creating a container Container string // Target is the build target for creating the worker image. Worker string @@ -1149,6 +1149,12 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot ctx := startTestSpan(baseCtx, t) testCustomLinuxWorker(ctx, t, testConfig.Target, testConfig.Worker) }) + + t.Run("pinned build dependencies", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + testPinnedBuildDeps(ctx, t, testConfig.Target, testConfig.Worker) + }) } func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetConfig, workerCfg workerConfig) { @@ -1242,6 +1248,114 @@ func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetCo }) } +func testPinnedBuildDeps(ctx context.Context, t *testing.T, targetCfg targetConfig, workerCfg workerConfig) { + var pkgName = "dalec-test-package" + + getTestPackageSpec := func(version string) *dalec.Spec { + depSpec := &dalec.Spec{ + Name: pkgName, + Version: version, + Revision: "1", + Description: "A basic package for various testing uses", + License: "MIT", + Sources: map[string]dalec.Source{ + "version.txt": { + Inline: &dalec.SourceInline{ + File: &dalec.SourceInlineFile{ + Contents: "version: " + version, + }, + }, + }, + }, + Artifacts: dalec.Artifacts{ + Docs: map[string]dalec.ArtifactConfig{ + "version.txt": {}, + }, + }, + } + + return depSpec + } + + depSpecs := []*dalec.Spec{ + getTestPackageSpec("1.1.1"), + getTestPackageSpec("1.2.0"), + getTestPackageSpec("1.3.0"), + } + + spec := &dalec.Spec{ + Name: "dalec-test-pinned-build-deps", + Version: "0.0.1", + Revision: "1", + Description: "Testing allowing custom worker images to be provided", + License: "MIT", + } + + tests := []struct { + name string + constraints string + want string + }{ + { + name: "exact dep available", + constraints: "== 1.1.1", + want: "1.1.1", + }, + + { + name: "lt dep available", + constraints: "< 1.3.0", + want: "1.2.0", + }, + + { + name: "gt dep available", + constraints: "> 1.2.0", + want: "1.3.0", + }, + } + + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { + // Build the worker target, this will give us the worker image as an output. + // Note: Currently we need to provide a dalec spec just due to how the router is setup. + // The spec can be nil, though, it just needs to be parsable by yaml unmarshaller. + sr := newSolveRequest(withBuildTarget(targetCfg.Worker), withSpec(ctx, t, nil)) + worker := reqToState(ctx, gwc, sr, t) + + var pkgs []llb.State + for _, depSpec := range depSpecs { + sr := newSolveRequest(withSpec(ctx, t, depSpec), withBuildTarget(targetCfg.Package)) + pkg := reqToState(ctx, gwc, sr, t) + pkgs = append(pkgs, pkg) + } + worker = worker.With(workerCfg.CreateRepo(llb.Merge(pkgs))) + + for _, tt := range tests { + spec.Dependencies = &dalec.PackageDependencies{ + Build: map[string]dalec.PackageConstraints{ + pkgName: { + Version: []string{tt.constraints}, + }, + }, + } + + spec.Build.Steps = []dalec.BuildStep{ + { + Command: fmt.Sprintf(`[[ $(cat /usr/share/doc/%s/version.txt) == "version: %s" ]]`, pkgName, tt.want), + }, + } + + sr = newSolveRequest(withSpec(ctx, t, spec), withBuildContext(ctx, t, workerCfg.ContextName, worker), withBuildTarget(targetCfg.Container)) + res := solveT(ctx, t, gwc, sr) + _, err := res.SingleRef() + + if err != nil { + t.Fatal(err) + } + } + }) +} + func validatePathAndPermissions(ctx context.Context, ref gwclient.Reference, path string, expected os.FileMode) error { stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: path}) if err != nil {