From 4cfa10860637c0ce6f0156b4e61708300eefb1d7 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 11 Dec 2023 11:27:06 -0500 Subject: [PATCH 01/39] Add Source data structures The source variants, with all sub-fields, have been added to the `Source` struct. Note that this code does not yet work, and will not compile. Signed-off-by: Peter Engelbert --- frontend/gateway.go | 6 +- load.go | 2 +- source.go | 4 +- spec.go | 131 +++++++++++++++++++++++++------------------- 4 files changed, 82 insertions(+), 61 deletions(-) diff --git a/frontend/gateway.go b/frontend/gateway.go index 0ce7dbaf2..1b4ac27d5 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -25,7 +25,7 @@ const ( dalecSubrequstForwardBuild = "dalec.forward.build" ) -func getDockerfile(ctx context.Context, client gwclient.Client, spec *dalec.BuildSpec, defPb *pb.Definition) ([]byte, error) { +func getDockerfile(ctx context.Context, client gwclient.Client, spec *dalec.SourceBuild, defPb *pb.Definition) ([]byte, error) { if spec.Inline != "" { return []byte(spec.Inline), nil } @@ -60,9 +60,9 @@ func getDockerfile(ctx context.Context, client gwclient.Client, spec *dalec.Buil // ForwarderFromClient creates a [dalec.ForwarderFunc] from a gateway client. // This is used for forwarding builds to other frontends in [dalec.Source2LLBGetter] func ForwarderFromClient(ctx context.Context, client gwclient.Client) dalec.ForwarderFunc { - return func(st llb.State, spec *dalec.BuildSpec) (llb.State, error) { + return func(st llb.State, spec *dalec.SourceBuild) (llb.State, error) { if spec == nil { - spec = &dalec.BuildSpec{} + spec = &dalec.SourceBuild{} } if spec.File != "" && spec.Inline != "" { diff --git a/load.go b/load.go index 1f4a633c3..c44ae4dbf 100644 --- a/load.go +++ b/load.go @@ -214,7 +214,7 @@ func (c *CheckOutput) processBuildArgs(lex *shell.Lex, args map[string]string) e } func (c *TestSpec) processBuildArgs(lex *shell.Lex, args map[string]string, name string) error { - if err := c.CmdSpec.processBuildArgs(lex, args, name); err != nil { + if err := c.SourceCommand.processBuildArgs(lex, args, name); err != nil { return err } diff --git a/source.go b/source.go index b21d9ed05..769eb253f 100644 --- a/source.go +++ b/source.go @@ -23,7 +23,7 @@ const ( type LLBGetter func(sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) -type ForwarderFunc func(llb.State, *BuildSpec) (llb.State, error) +type ForwarderFunc func(llb.State, *SourceBuild) (llb.State, error) type SourceOpts struct { Resolver llb.ImageMetaResolver @@ -36,7 +36,7 @@ func shArgs(cmd string) llb.RunOption { } // must not be called with a nil cmd pointer -func generateSourceFromImage(s *Spec, name string, st llb.State, cmd *CmdSpec, sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.ExecState, error) { +func generateSourceFromImage(s *Spec, name string, st llb.State, cmd *SourceCommand, sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.ExecState, error) { var zero llb.ExecState if len(cmd.Steps) == 0 { return zero, fmt.Errorf("no steps defined for image source") diff --git a/spec.go b/spec.go index d9699fdf6..cf68a0b8a 100644 --- a/spec.go +++ b/spec.go @@ -179,14 +179,84 @@ type ImageConfig struct { Base string `yaml:"base,omitempty" json:"base,omitempty"` } +type SourceDockerImage struct { + Ref string `yaml:"ref" json:"ref"` +} + +type SourceGit struct { + Ref string `yaml:"ref" json:"ref"` + KeepGitDir bool `yaml:"keepGitDir" json:"keepGitDir"` +} + +// No longer supports `.git` URLs as git repos. That has to be done with +// `SourceGit` +type SourceHTTPS struct { + Ref string `yaml:"ref" json:"ref"` +} + +type SourceContext struct { + Ref string `yaml:"ref" json:"ref"` +} + +// i.e. just rename `BuildSpec` to `SourceBuild` +// SourceBuild is used to generate source from a build. +type SourceBuild struct { + // Target specifies the build target to use. + // If unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default). + Target string `yaml:"target,omitempty" json:"target,omitempty"` + // Args are the build args to pass to the build. + Args map[string]string `yaml:"args,omitempty" json:"args,omitempty"` + // File is the path to the build file in the build context + // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. + // This is exclusive with [Inline] + File string `yaml:"file,omitempty" json:"file,omitempty"` + + // Inline is an inline build spec to use. + // This can be used to specify a dockerfile instead of using one in the build context + // This is exclusive with [File] + Inline string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` +} + +type SourceAnotherSource struct { + Ref string `yaml:"ref" json:"ref"` +} + +// SourceCommand is used to execute a command to generate a source from a docker image. +type SourceCommand struct { + // Dir is the working directory to run the command in. + Dir string `yaml:"dir,omitempty" json:"dir,omitempty"` + + // Mounts is the list of sources to mount into the build steps. + Mounts []SourceMount `yaml:"mounts,omitempty" json:"mounts,omitempty"` + + // List of CacheDirs which will be used across all Steps + CacheDirs map[string]CacheDirConfig `yaml:"cache_dirs,omitempty" json:"cache_dirs,omitempty"` + + // Env is the list of environment variables to set for all commands in this step group. + Env map[string]string `yaml:"env,omitempty" json:"env,omitempty"` + + // Steps is the list of commands to run to generate the source. + // Steps are run sequentially and results of each step should be cached. + Steps []*BuildStep `yaml:"steps" json:"steps" jsonschema:"required"` +} + // Source defines a source to be used in the build. // A source can be a local directory, a git repositoryt, http(s) URL, etc. type Source struct { - // Ref is a unique identifier for the source. - // Example: "docker-image://busybox:latest", "https://github.com/moby/buildkit.git#master", "context:// + // This is an embedded union representing all of the possible source types. + // Only one should be non-nil at any given time. It is considered an error + // condition if more than one is non-nil, or if all are nil. // - // When a source is specified as part of a [CmdSpec], it may also be used to reference a top-level source. - Ref string `yaml:"ref" json:"ref" jsonschema:"required"` + // === Begin Source Variants === + DockerImage *SourceDockerImage `yaml:"image,omitempty" json:"image,omitempty"` + Git *SourceGit `yaml:"git,omitempty" json:"git,omitempty"` + HTTPS *SourceHTTPS `yaml:"https,omitempty" json:"https,omitempty"` + Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` + Build *SourceBuild `yaml:"build,omitempty" json:"build,omitempty"` + Source *SourceAnotherSource `yaml:"source,omitempty" json:"source,omitempty"` + Cmd *SourceCommand `yaml:"cmd,omitempty" json:"cmd,omitempty"` + // === End Source Variants === + // Path is the path to the source after fetching it based on the identifier. Path string `yaml:"path,omitempty" json:"path,omitempty"` @@ -198,17 +268,6 @@ type Source struct { // KeepGitDir is used to keep the .git directory after fetching the source for git references. KeepGitDir bool `yaml:"keep_git_dir,omitempty" json:"keep_git_dir,omitempty"` - - // Cmd is used to generate the source from a command. - // This can be used when Ref is "docker-image://" - // If ref is "cmd://", this is required. - Cmd *CmdSpec `yaml:"cmd,omitempty" json:"cmd,omitempty"` - - // Build is used to generate source from a build. - // This is used when [Ref]` is "build://" - // The context for the build is assumed too be specified in after `build://` in the ref, e.g. `build://https://github.com/moby/buildkit.git#master` - // When nothing is specified after `build://`, the context is assumed to be the current build context. - Build *BuildSpec `yaml:"build,omitempty" json:"build,omitempty"` } func (Source) JSONSchemaExtend(schema *jsonschema.Schema) { @@ -227,25 +286,6 @@ func (Source) JSONSchemaExtend(schema *jsonschema.Schema) { } } -// BuildSpec is used to generate source from a build. -// This is used when [Source.Ref] is "build://" to forward a build (aka a nested build) through to buildkit. -type BuildSpec struct { - // Target specifies the build target to use. - // If unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default). - Target string `yaml:"target,omitempty" json:"target,omitempty"` - // Args are the build args to pass to the build. - Args map[string]string `yaml:"args,omitempty" json:"args,omitempty"` - // File is the path to the build file in the build context - // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. - // This is exclusive with [Inline] - File string `yaml:"file,omitempty" json:"file,omitempty"` - - // Inline is an inline build spec to use. - // This can be used to specify a dockerfile instead of using one in the build context - // This is exclusive with [File] - Inline string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` -} - // PackageDependencies is a list of dependencies for a package. // This will be included in the package metadata so that the package manager can install the dependencies. // It also includes build-time dedendencies, which we'll install before running any build steps. @@ -288,25 +328,6 @@ type SourceMount struct { Spec Source `yaml:"spec" json:"spec" jsonschema:"required"` } -// CmdSpec is used to execute a command to generate a source from a docker image. -type CmdSpec struct { - // Dir is the working directory to run the command in. - Dir string `yaml:"dir,omitempty" json:"dir,omitempty"` - - // Mounts is the list of sources to mount into the build steps. - Mounts []SourceMount `yaml:"mounts,omitempty" json:"mounts,omitempty"` - - // List of CacheDirs which will be used across all Steps - CacheDirs map[string]CacheDirConfig `yaml:"cache_dirs,omitempty" json:"cache_dirs,omitempty"` - - // Env is the list of environment variables to set for all commands in this step group. - Env map[string]string `yaml:"env,omitempty" json:"env,omitempty"` - - // Steps is the list of commands to run to generate the source. - // Steps are run sequentially and results of each step should be cached. - Steps []*BuildStep `yaml:"steps" json:"steps" jsonschema:"required"` -} - // CacheDirConfig configures a persistent cache to be used across builds. type CacheDirConfig struct { // Mode is the locking mode to set on the cache directory @@ -367,8 +388,8 @@ type Target struct { type TestSpec struct { // Name is the name of the test // This will be used to output the test results - Name string `yaml:"name" json:"name" jsonschema:"required"` - CmdSpec `yaml:",inline"` + Name string `yaml:"name" json:"name" jsonschema:"required"` + SourceCommand `yaml:",inline"` // Steps is the list of commands to run to test the package. Steps []TestStep `yaml:"steps" json:"steps" jsonschema:"required"` // Files is the list of files to check after running the steps. From 5a22b1c86c7eaccb98cc529a46da95aefc07e364 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 11 Dec 2023 11:49:42 -0500 Subject: [PATCH 02/39] Add `GetRef` and `SetRef` methods to `Source` These are basic getters and setters for the `Ref` subfields on various `Source` variants. Because any one of these variants could be non-nil, we first have to determine which is in use. Not all Source vaiants have a `.Ref` field, so those cases need to be handled as well. These methods each return an additional `bool` value depending on whether or not the `.Ref` field exists in the Source variant. Signed-off-by: Peter Engelbert --- frontend/rpm/handle_sources.go | 7 +++- load.go | 71 +++++++++++++++++++++++++++++++--- spec.go | 1 + 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index 24fd53933..b0049fcfe 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -88,7 +88,12 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err return nil, err } - pg := llb.ProgressGroup(pgID, "Add spec source: "+k+" "+src.Ref, false) + ref, ok := src.GetRef() + if !ok { + continue + } + + pg := llb.ProgressGroup(pgID, "Add spec source: "+k+" "+ref, false) st, err := dalec.Source2LLBGetter(spec, src, k)(sOpt, pg) if err != nil { return nil, err diff --git a/load.go b/load.go index c44ae4dbf..6d8d3af07 100644 --- a/load.go +++ b/load.go @@ -22,6 +22,59 @@ func knownArg(key string) bool { const DefaultPatchStrip int = 1 +// GetRef determines the Source variant and returns the value of the `Ref` +// field, and the second return value will be `true`. If the variant doesn't +// have a `Ref` field, the second return value will be `false`. +func (s *Source) GetRef() (string, bool) { + switch { + case s.DockerImage != nil: + return s.DockerImage.Ref, true + case s.Git != nil: + return s.Git.Ref, true + case s.HTTPS != nil: + return s.HTTPS.Ref, true + case s.Context != nil: + return s.Context.Ref, true + case s.Build != nil: + return s.Build.Ref, true + case s.Source != nil: + return s.Source.Ref, true + case s.Cmd != nil: + return "", false + default: + return "", false + } +} + +// SetRef determines the Source variant and sets the `Ref` field. If the Source +// variant doesn't have a `Ref` field, SetRef does nothing and returns `false`. +func (s *Source) SetRef(ref string) bool { + switch { + case s.DockerImage != nil: + s.DockerImage.Ref = ref + return true + case s.Git != nil: + s.Git.Ref = ref + return true + case s.HTTPS != nil: + s.HTTPS.Ref = ref + return true + case s.Context != nil: + s.Context.Ref = ref + return true + case s.Build != nil: + s.Build.Ref = ref + return true + case s.Source != nil: + s.Source.Ref = ref + return true + case s.Cmd != nil: + return false + default: + return false + } +} + // LoadSpec loads a spec from the given data. // env is a map of environment variables to use for shell-style expansion in the spec. func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { @@ -46,11 +99,15 @@ func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { } for name, src := range spec.Sources { - updated, err := lex.ProcessWordWithMap(src.Ref, args) + ref, ok := src.GetRef() + if !ok { + continue + } + updated, err := lex.ProcessWordWithMap(ref, args) if err != nil { return nil, fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) } - src.Ref = updated + _ = src.SetRef(updated) if err := src.Cmd.processBuildArgs(lex, args, name); err != nil { return nil, fmt.Errorf("error performing shell expansion on source %q: %w", name, err) } @@ -124,16 +181,20 @@ func (s *BuildStep) processBuildArgs(lex *shell.Lex, args map[string]string, i i return nil } -func (c *CmdSpec) processBuildArgs(lex *shell.Lex, args map[string]string, name string) error { +func (c *SourceCommand) processBuildArgs(lex *shell.Lex, args map[string]string, name string) error { if c == nil { return nil } for i, smnt := range c.Mounts { - updated, err := lex.ProcessWordWithMap(smnt.Spec.Ref, args) + ref, ok := smnt.Spec.GetRef() + if !ok { + continue + } + updated, err := lex.ProcessWordWithMap(ref, args) if err != nil { return fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) } - c.Mounts[i].Spec.Ref = updated + c.Mounts[i].Spec.SetRef(updated) } for k, v := range c.Env { updated, err := lex.ProcessWordWithMap(v, args) diff --git a/spec.go b/spec.go index cf68a0b8a..76c774700 100644 --- a/spec.go +++ b/spec.go @@ -201,6 +201,7 @@ type SourceContext struct { // i.e. just rename `BuildSpec` to `SourceBuild` // SourceBuild is used to generate source from a build. type SourceBuild struct { + Ref string `yaml:"ref" json:"ref"` // Target specifies the build target to use. // If unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default). Target string `yaml:"target,omitempty" json:"target,omitempty"` From 91022b684043fc61ff4d111ea4b67e9e143b6b4a Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 11 Dec 2023 12:21:27 -0500 Subject: [PATCH 03/39] Switch on embedded union variant Change switch on scheme to union variant in `source2LLBGetter` (`source.go`). Signed-off-by: Peter Engelbert --- source.go | 61 ++++++++++++++++++++++++------------------------------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/source.go b/source.go index 769eb253f..f6b186306 100644 --- a/source.go +++ b/source.go @@ -7,6 +7,7 @@ import ( "io" "strings" + "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerui" "github.com/moby/buildkit/identity" @@ -87,11 +88,6 @@ func Source2LLBGetter(s *Spec, src Source, name string) LLBGetter { func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter { return func(sOpt SourceOpts, opts ...llb.ConstraintsOpt) (ret llb.State, retErr error) { - scheme, ref, err := SplitSourceRef(src.Ref) - if err != nil { - return llb.Scratch(), err - } - var ( includeExcludeHandled bool pathHandled bool @@ -136,9 +132,11 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter ) }() - switch scheme { - case sourcetypes.DockerImageScheme: - st := llb.Image(ref, llb.WithMetaResolver(sOpt.Resolver), withConstraints(opts)) + // sourceType, err := src.GetSourceKind() + switch { + case src.DockerImage != nil: + img := src.DockerImage + st := llb.Image(img.Ref, llb.WithMetaResolver(sOpt.Resolver), withConstraints(opts)) if src.Cmd == nil { return st, nil @@ -153,9 +151,10 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter return eSt.AddMount(src.Path, llb.Scratch()), nil } return eSt.Root(), nil - case sourcetypes.GitScheme: + case src.Git != nil: + git := src.Git // TODO: Pass git secrets - ref, err := gitutil.ParseGitRef(ref) + ref, err := gitutil.ParseGitRef(git.Ref) if err != nil { return llb.Scratch(), fmt.Errorf("could not parse git ref: %w", err) } @@ -166,39 +165,32 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } gOpts = append(gOpts, withConstraints(opts)) return llb.Git(ref.Remote, ref.Commit, gOpts...), nil - case sourcetypes.HTTPScheme, sourcetypes.HTTPSScheme: - ref, err := gitutil.ParseGitRef(src.Ref) - if err == nil { - // TODO: Pass git secrets - var gOpts []llb.GitOption - if src.KeepGitDir { - gOpts = append(gOpts, llb.KeepGitDir()) - } - gOpts = append(gOpts, withConstraints(opts)) - return llb.Git(ref.Remote, ref.Commit, gOpts...), nil - } else { - opts := []llb.HTTPOption{withConstraints(opts)} - opts = append(opts, llb.Filename(name)) - return llb.HTTP(src.Ref, opts...), nil - } - case sourceTypeContext: + case src.HTTPS != nil: + https := src.HTTPS + opts := []llb.HTTPOption{withConstraints(opts)} + opts = append(opts, llb.Filename(name)) + return llb.HTTP(https.Ref, opts...), nil + case src.Context != nil: + srcCtx := src.Context st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) if err != nil { return llb.Scratch(), err } includeExcludeHandled = true - if src.Path == "" && ref != "" { - src.Path = ref + if src.Path == "" && srcCtx.Ref != "" { + src.Path = srcCtx.Ref } return *st, nil - case sourceTypeBuild: + case src.Build != nil: + var err error + build := src.Build var st llb.State - if ref == "" { + if build.Ref == "" { st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) } else { src2 := Source{ - Ref: ref, + Build: &dalec.SourceBuild{Ref: build.Ref}, Path: src.Path, Includes: src.Includes, Excludes: src.Excludes, @@ -212,11 +204,12 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } return sOpt.Forward(st, src.Build) - case sourceTypeSource: - src := s.Sources[ref] + case src.Source != nil: + srcSrc := src.Source + src := s.Sources[srcSrc.Ref] return source2LLBGetter(s, src, name, forMount)(sOpt, opts...) default: - return llb.Scratch(), fmt.Errorf("unsupported source type: %s", scheme) + return llb.Scratch(), fmt.Errorf("No source variant found") } } } From c80430061517812f30956a8f626facfd4f52d243 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 11 Dec 2023 12:27:37 -0500 Subject: [PATCH 04/39] Implement switching logic on `Source` variants Continue implementing the necessary switches. This code compiles. Signed-off-by: Peter Engelbert --- source.go | 78 +++++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 52 deletions(-) diff --git a/source.go b/source.go index f6b186306..ec18e06a8 100644 --- a/source.go +++ b/source.go @@ -7,11 +7,9 @@ import ( "io" "strings" - "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerui" "github.com/moby/buildkit/identity" - sourcetypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/gitutil" ) @@ -190,7 +188,7 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) } else { src2 := Source{ - Build: &dalec.SourceBuild{Ref: build.Ref}, + Build: &SourceBuild{Ref: build.Ref}, Path: src.Path, Includes: src.Includes, Excludes: src.Excludes, @@ -242,23 +240,16 @@ func WithCreateDestPath() llb.CopyOption { } func SourceIsDir(src Source) (bool, error) { - scheme, _, err := SplitSourceRef(src.Ref) - if err != nil { - return false, err - } - switch scheme { - case sourcetypes.DockerImageScheme, - sourcetypes.GitScheme, - sourceTypeBuild, - sourceTypeContext: + switch { + case src.DockerImage != nil, + src.Git != nil, + src.Build != nil, + src.Context != nil: return true, nil - case sourcetypes.HTTPScheme, sourcetypes.HTTPSScheme: - if isGitRef(src.Ref) { - return true, nil - } + case src.HTTPS != nil: return false, nil default: - return false, fmt.Errorf("unsupported source type: %s", scheme) + return false, fmt.Errorf("unsupported source type") } } @@ -272,20 +263,16 @@ func isGitRef(ref string) bool { // so that others can reproduce the build. func (s Source) Doc() (io.Reader, error) { b := bytes.NewBuffer(nil) - scheme, ref, err := SplitSourceRef(s.Ref) - if err != nil { - return nil, err - } - - switch scheme { - case sourceTypeSource: - fmt.Fprintln(b, "Generated from another source named:", ref) - case sourceTypeContext: + switch { + case s.Source != nil: + fmt.Fprintln(b, "Generated from another source named:", s.Source.Ref) + case s.Context != nil: fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") - case sourceTypeBuild: + case s.Build != nil: + build := s.Build fmt.Fprintln(b, "Generated from a docker build:") fmt.Fprintln(b, " Docker Build Target:", s.Build.Target) - fmt.Fprintln(b, " Docker Build Ref:", ref) + fmt.Fprintln(b, " Docker Build Ref:", build.Ref) if len(s.Build.Args) > 0 { sorted := SortMapKeys(s.Build.Args) @@ -312,44 +299,31 @@ func (s Source) Doc() (io.Reader, error) { } fmt.Fprintln(b, " Dockerfile path in context:", p) } - case sourcetypes.HTTPScheme, sourcetypes.HTTPSScheme: - ref, err := gitutil.ParseGitRef(ref) - if err == nil { - // git ref - fmt.Fprintln(b, "Generated from a git repository:") - fmt.Fprintln(b, " Remote:", scheme+"://"+ref.Remote) - fmt.Fprintln(b, " Ref:", ref.Commit) - if ref.SubDir != "" { - fmt.Fprintln(b, " Subdir:", ref.SubDir) - } - if s.Path != "" { - fmt.Fprintln(b, " Extraced path:", s.Path) - } - } else { - fmt.Fprintln(b, "Generated from a http(s) source:") - fmt.Fprintln(b, " URL:", ref) - } - case sourcetypes.GitScheme: - ref, err := gitutil.ParseGitRef(ref) + case s.HTTPS != nil: + fmt.Fprintln(b, "Generated from a http(s) source:") + fmt.Fprintln(b, " URL:", s.HTTPS.Ref) + case s.Git != nil: + git := s.Git + ref, err := gitutil.ParseGitRef(git.Ref) if err != nil { return nil, err } fmt.Fprintln(b, "Generated from a git repository:") - fmt.Fprintln(b, " Remote:", scheme+"://"+ref.Remote) fmt.Fprintln(b, " Ref:", ref.Commit) if s.Path != "" { fmt.Fprintln(b, " Extraced path:", s.Path) } - case sourcetypes.DockerImageScheme: + case s.DockerImage != nil: + img := s.DockerImage if s.Cmd == nil { fmt.Fprintln(b, "Generated from a docker image:") - fmt.Fprintln(b, " Image:", ref) + fmt.Fprintln(b, " Image:", img.Ref) if s.Path != "" { fmt.Fprintln(b, " Extraced path:", s.Path) } } else { fmt.Fprintln(b, "Generated from running a command(s) in a docker image:") - fmt.Fprintln(b, " Image:", ref) + fmt.Fprintln(b, " Image:", img.Ref) if s.Path != "" { fmt.Fprintln(b, " Extraced path:", s.Path) } @@ -398,7 +372,7 @@ func (s Source) Doc() (io.Reader, error) { default: // This should be unrecable. // We could panic here, but ultimately this is just a doc string and parsing user generated content. - fmt.Fprintln(b, "Generated from an unknown source type:", s.Ref) + fmt.Fprintln(b, "Generated from an unknown source type") } return b, nil From f0036cf482de286e2065053f569bbc6e69065148 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 11 Dec 2023 13:14:21 -0500 Subject: [PATCH 05/39] Remove `KeepGitDir` field from `Source`. This field is now found on `Source.Git`. Signed-off-by: Peter Engelbert --- source.go | 13 ++++++------- spec.go | 3 --- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/source.go b/source.go index ec18e06a8..151736dfb 100644 --- a/source.go +++ b/source.go @@ -158,7 +158,7 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } var gOpts []llb.GitOption - if src.KeepGitDir { + if git.KeepGitDir { gOpts = append(gOpts, llb.KeepGitDir()) } gOpts = append(gOpts, withConstraints(opts)) @@ -188,12 +188,11 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) } else { src2 := Source{ - Build: &SourceBuild{Ref: build.Ref}, - Path: src.Path, - Includes: src.Includes, - Excludes: src.Excludes, - KeepGitDir: src.KeepGitDir, - Cmd: src.Cmd, + Build: &SourceBuild{Ref: build.Ref}, + Path: src.Path, + Includes: src.Includes, + Excludes: src.Excludes, + Cmd: src.Cmd, } st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) if err != nil { diff --git a/spec.go b/spec.go index 76c774700..22751b080 100644 --- a/spec.go +++ b/spec.go @@ -266,9 +266,6 @@ type Source struct { Includes []string `yaml:"includes,omitempty" json:"includes,omitempty"` // Excludes is a list of paths underneath `Path` to exclude, everything else is included Excludes []string `yaml:"excludes,omitempty" json:"excludes,omitempty"` - - // KeepGitDir is used to keep the .git directory after fetching the source for git references. - KeepGitDir bool `yaml:"keep_git_dir,omitempty" json:"keep_git_dir,omitempty"` } func (Source) JSONSchemaExtend(schema *jsonschema.Schema) { From a51da2769eefb746d07389d0b0531e934543474a Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 12 Dec 2023 11:07:59 -0500 Subject: [PATCH 06/39] Rename fields on some Source variants Also, remove `Source.Source` and `SourceAnotherSource`. Signed-off-by: Peter Engelbert --- load.go | 21 ++++++++------------- source.go | 24 +++++++++--------------- spec.go | 26 +++++++++++--------------- 3 files changed, 28 insertions(+), 43 deletions(-) diff --git a/load.go b/load.go index 6d8d3af07..1f6828dfe 100644 --- a/load.go +++ b/load.go @@ -30,15 +30,13 @@ func (s *Source) GetRef() (string, bool) { case s.DockerImage != nil: return s.DockerImage.Ref, true case s.Git != nil: - return s.Git.Ref, true + return s.Git.URL, true case s.HTTPS != nil: - return s.HTTPS.Ref, true + return s.HTTPS.URL, true case s.Context != nil: - return s.Context.Ref, true + return s.Context.Name, true case s.Build != nil: - return s.Build.Ref, true - case s.Source != nil: - return s.Source.Ref, true + return s.Build.Name, true case s.Cmd != nil: return "", false default: @@ -54,19 +52,16 @@ func (s *Source) SetRef(ref string) bool { s.DockerImage.Ref = ref return true case s.Git != nil: - s.Git.Ref = ref + s.Git.URL = ref return true case s.HTTPS != nil: - s.HTTPS.Ref = ref + s.HTTPS.URL = ref return true case s.Context != nil: - s.Context.Ref = ref + s.Context.Name = ref return true case s.Build != nil: - s.Build.Ref = ref - return true - case s.Source != nil: - s.Source.Ref = ref + s.Build.Name = ref return true case s.Cmd != nil: return false diff --git a/source.go b/source.go index 151736dfb..891697196 100644 --- a/source.go +++ b/source.go @@ -152,7 +152,7 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter case src.Git != nil: git := src.Git // TODO: Pass git secrets - ref, err := gitutil.ParseGitRef(git.Ref) + ref, err := gitutil.ParseGitRef(git.URL) if err != nil { return llb.Scratch(), fmt.Errorf("could not parse git ref: %w", err) } @@ -167,7 +167,7 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter https := src.HTTPS opts := []llb.HTTPOption{withConstraints(opts)} opts = append(opts, llb.Filename(name)) - return llb.HTTP(https.Ref, opts...), nil + return llb.HTTP(https.URL, opts...), nil case src.Context != nil: srcCtx := src.Context st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) @@ -176,19 +176,19 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } includeExcludeHandled = true - if src.Path == "" && srcCtx.Ref != "" { - src.Path = srcCtx.Ref + if src.Path == "" && srcCtx.Name != "" { + src.Path = srcCtx.Name } return *st, nil case src.Build != nil: var err error build := src.Build var st llb.State - if build.Ref == "" { + if build.Name == "" { st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) } else { src2 := Source{ - Build: &SourceBuild{Ref: build.Ref}, + Build: &SourceBuild{Name: build.Name}, Path: src.Path, Includes: src.Includes, Excludes: src.Excludes, @@ -201,10 +201,6 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } return sOpt.Forward(st, src.Build) - case src.Source != nil: - srcSrc := src.Source - src := s.Sources[srcSrc.Ref] - return source2LLBGetter(s, src, name, forMount)(sOpt, opts...) default: return llb.Scratch(), fmt.Errorf("No source variant found") } @@ -263,15 +259,13 @@ func isGitRef(ref string) bool { func (s Source) Doc() (io.Reader, error) { b := bytes.NewBuffer(nil) switch { - case s.Source != nil: - fmt.Fprintln(b, "Generated from another source named:", s.Source.Ref) case s.Context != nil: fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") case s.Build != nil: build := s.Build fmt.Fprintln(b, "Generated from a docker build:") fmt.Fprintln(b, " Docker Build Target:", s.Build.Target) - fmt.Fprintln(b, " Docker Build Ref:", build.Ref) + fmt.Fprintln(b, " Docker Build Ref:", build.Name) if len(s.Build.Args) > 0 { sorted := SortMapKeys(s.Build.Args) @@ -300,10 +294,10 @@ func (s Source) Doc() (io.Reader, error) { } case s.HTTPS != nil: fmt.Fprintln(b, "Generated from a http(s) source:") - fmt.Fprintln(b, " URL:", s.HTTPS.Ref) + fmt.Fprintln(b, " URL:", s.HTTPS.URL) case s.Git != nil: git := s.Git - ref, err := gitutil.ParseGitRef(git.Ref) + ref, err := gitutil.ParseGitRef(git.URL) if err != nil { return nil, err } diff --git a/spec.go b/spec.go index 22751b080..badb56065 100644 --- a/spec.go +++ b/spec.go @@ -184,24 +184,25 @@ type SourceDockerImage struct { } type SourceGit struct { - Ref string `yaml:"ref" json:"ref"` + URL string `yaml:"url" json:"url"` + Commit string `yaml:"commit" json:"commit"` KeepGitDir bool `yaml:"keepGitDir" json:"keepGitDir"` } // No longer supports `.git` URLs as git repos. That has to be done with // `SourceGit` type SourceHTTPS struct { - Ref string `yaml:"ref" json:"ref"` + URL string `yaml:"url" json:"url"` } type SourceContext struct { - Ref string `yaml:"ref" json:"ref"` + Name string `yaml:"name" json:"name"` } // i.e. just rename `BuildSpec` to `SourceBuild` // SourceBuild is used to generate source from a build. type SourceBuild struct { - Ref string `yaml:"ref" json:"ref"` + Name string `yaml:"name" json:"name"` // Target specifies the build target to use. // If unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default). Target string `yaml:"target,omitempty" json:"target,omitempty"` @@ -218,10 +219,6 @@ type SourceBuild struct { Inline string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` } -type SourceAnotherSource struct { - Ref string `yaml:"ref" json:"ref"` -} - // SourceCommand is used to execute a command to generate a source from a docker image. type SourceCommand struct { // Dir is the working directory to run the command in. @@ -249,13 +246,12 @@ type Source struct { // condition if more than one is non-nil, or if all are nil. // // === Begin Source Variants === - DockerImage *SourceDockerImage `yaml:"image,omitempty" json:"image,omitempty"` - Git *SourceGit `yaml:"git,omitempty" json:"git,omitempty"` - HTTPS *SourceHTTPS `yaml:"https,omitempty" json:"https,omitempty"` - Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` - Build *SourceBuild `yaml:"build,omitempty" json:"build,omitempty"` - Source *SourceAnotherSource `yaml:"source,omitempty" json:"source,omitempty"` - Cmd *SourceCommand `yaml:"cmd,omitempty" json:"cmd,omitempty"` + DockerImage *SourceDockerImage `yaml:"image,omitempty" json:"image,omitempty"` + Git *SourceGit `yaml:"git,omitempty" json:"git,omitempty"` + HTTPS *SourceHTTPS `yaml:"https,omitempty" json:"https,omitempty"` + Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` + Build *SourceBuild `yaml:"build,omitempty" json:"build,omitempty"` + Cmd *SourceCommand `yaml:"cmd,omitempty" json:"cmd,omitempty"` // === End Source Variants === // Path is the path to the source after fetching it based on the identifier. From 945ff2f465797537185e6f00b025f0c654b25e57 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 15 Dec 2023 10:27:58 -0500 Subject: [PATCH 07/39] Implement source.processArgs * Remove source.GetRef and source.SetRef * Add source.processArgs and use it where necessary Signed-off-by: Peter Engelbert --- frontend/rpm/handle_sources.go | 18 ++++++-- load.go | 76 +++++++++++----------------------- 2 files changed, 39 insertions(+), 55 deletions(-) diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index b0049fcfe..565406ca5 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -88,12 +88,22 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err return nil, err } - ref, ok := src.GetRef() - if !ok { - continue + s := "" + switch { + case src.DockerImage != nil: + s = src.DockerImage.Ref + case src.Git != nil: + s = src.Git.URL + case src.HTTPS != nil: + s = src.HTTPS.URL + case src.Context != nil: + s = src.Context.Name + case src.Build != nil: + s = src.Build.Name + default: } - pg := llb.ProgressGroup(pgID, "Add spec source: "+k+" "+ref, false) + pg := llb.ProgressGroup(pgID, "Add spec source: "+k+" "+s, false) st, err := dalec.Source2LLBGetter(spec, src, k)(sOpt, pg) if err != nil { return nil, err diff --git a/load.go b/load.go index 1f6828dfe..51245bfc5 100644 --- a/load.go +++ b/load.go @@ -22,52 +22,38 @@ func knownArg(key string) bool { const DefaultPatchStrip int = 1 -// GetRef determines the Source variant and returns the value of the `Ref` -// field, and the second return value will be `true`. If the variant doesn't -// have a `Ref` field, the second return value will be `false`. -func (s *Source) GetRef() (string, bool) { +func (s *Source) processArgs(args map[string]string) error { + lex := shell.NewLex('\\') + + var sub *string switch { case s.DockerImage != nil: - return s.DockerImage.Ref, true + sub = &s.DockerImage.Ref case s.Git != nil: - return s.Git.URL, true + sub = &s.Git.URL case s.HTTPS != nil: - return s.HTTPS.URL, true + sub = &s.HTTPS.URL case s.Context != nil: - return s.Context.Name, true + sub = &s.Context.Name + if *sub == "" { + *sub = "context" + } case s.Build != nil: - return s.Build.Name, true - case s.Cmd != nil: - return "", false + sub = &s.Build.Name default: - return "", false } -} -// SetRef determines the Source variant and sets the `Ref` field. If the Source -// variant doesn't have a `Ref` field, SetRef does nothing and returns `false`. -func (s *Source) SetRef(ref string) bool { - switch { - case s.DockerImage != nil: - s.DockerImage.Ref = ref - return true - case s.Git != nil: - s.Git.URL = ref - return true - case s.HTTPS != nil: - s.HTTPS.URL = ref - return true - case s.Context != nil: - s.Context.Name = ref - return true - case s.Build != nil: - s.Build.Name = ref - return true - case s.Cmd != nil: - return false - default: - return false + if sub == nil { + return nil + } + + updated, err := lex.ProcessWordWithMap(*sub, args) + if err != nil { + return err } + + *sub = updated + return nil } // LoadSpec loads a spec from the given data. @@ -94,15 +80,9 @@ func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { } for name, src := range spec.Sources { - ref, ok := src.GetRef() - if !ok { - continue - } - updated, err := lex.ProcessWordWithMap(ref, args) - if err != nil { + if err := src.processArgs(args); err != nil { return nil, fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) } - _ = src.SetRef(updated) if err := src.Cmd.processBuildArgs(lex, args, name); err != nil { return nil, fmt.Errorf("error performing shell expansion on source %q: %w", name, err) } @@ -180,16 +160,10 @@ func (c *SourceCommand) processBuildArgs(lex *shell.Lex, args map[string]string, if c == nil { return nil } - for i, smnt := range c.Mounts { - ref, ok := smnt.Spec.GetRef() - if !ok { - continue - } - updated, err := lex.ProcessWordWithMap(ref, args) - if err != nil { + for _, s := range c.Mounts { + if err := s.Spec.processArgs(args); err != nil { return fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) } - c.Mounts[i].Spec.SetRef(updated) } for k, v := range c.Env { updated, err := lex.ProcessWordWithMap(v, args) From d0f681798f73b05e569b662846470bac9c07567a Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 15 Dec 2023 11:19:29 -0500 Subject: [PATCH 08/39] Add `SourceLocal` (local Source variant) It's often convenient to use files and directories from the local filesystem as a source. This commit essentially exposes `llb.Local(...)` to the user via the spec. This implementation may be incomplete, and should be reviewed. Signed-off-by: Peter Engelbert --- load.go | 2 ++ source.go | 5 +++++ spec.go | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/load.go b/load.go index 51245bfc5..aa6916673 100644 --- a/load.go +++ b/load.go @@ -40,6 +40,8 @@ func (s *Source) processArgs(args map[string]string) error { } case s.Build != nil: sub = &s.Build.Name + case s.Local != nil: + sub = &s.Local.Path default: } diff --git a/source.go b/source.go index 891697196..903227794 100644 --- a/source.go +++ b/source.go @@ -180,6 +180,9 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter src.Path = srcCtx.Name } return *st, nil + case src.Local != nil: + srcLocal := src.Local + return llb.Local(srcLocal.Path, localIncludeExcludeMerge(&src)), nil case src.Build != nil: var err error build := src.Build @@ -261,6 +264,8 @@ func (s Source) Doc() (io.Reader, error) { switch { case s.Context != nil: fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") + case s.Local != nil: + fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") case s.Build != nil: build := s.Build fmt.Fprintln(b, "Generated from a docker build:") diff --git a/spec.go b/spec.go index badb56065..dced06c0f 100644 --- a/spec.go +++ b/spec.go @@ -238,6 +238,10 @@ type SourceCommand struct { Steps []*BuildStep `yaml:"steps" json:"steps" jsonschema:"required"` } +type SourceLocal struct { + Path string `yaml:"path,omitempty" json:"path,omitempty"` +} + // Source defines a source to be used in the build. // A source can be a local directory, a git repositoryt, http(s) URL, etc. type Source struct { @@ -251,6 +255,7 @@ type Source struct { HTTPS *SourceHTTPS `yaml:"https,omitempty" json:"https,omitempty"` Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` Build *SourceBuild `yaml:"build,omitempty" json:"build,omitempty"` + Local *SourceLocal `yaml:"local,omitempty" json:"local,omitempty"` Cmd *SourceCommand `yaml:"cmd,omitempty" json:"cmd,omitempty"` // === End Source Variants === From f31c8d0299ce990fb4da23bdfa89b0152d155571 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 15 Dec 2023 13:07:36 -0500 Subject: [PATCH 09/39] Add validation functions Validate that a source does not have multiple variants defined, and that it has at least one variant defined. If there's a more elegant implementation, feel free to suggest it. Signed-off-by: Peter Engelbert --- load.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/load.go b/load.go index aa6916673..33221364a 100644 --- a/load.go +++ b/load.go @@ -58,6 +58,51 @@ func (s *Source) processArgs(args map[string]string) error { return nil } +func checkDuplicate[T any](c *int, p *T) error { + if c == nil { + return fmt.Errorf("validation function (check) called with nil pointer") + } + + if p != nil { + (*c)++ + } + + if *c > 1 { + return fmt.Errorf("more than one source variant defined") + } + + return nil +} + +func (s *Source) validate() error { + count := 0 + + if err := checkDuplicate(&count, s.DockerImage); err != nil { + return err + } + if err := checkDuplicate(&count, s.Git); err != nil { + return err + } + if err := checkDuplicate(&count, s.HTTPS); err != nil { + return err + } + if err := checkDuplicate(&count, s.Context); err != nil { + return err + } + if err := checkDuplicate(&count, s.Build); err != nil { + return err + } + if err := checkDuplicate(&count, s.Local); err != nil { + return err + } + + if count == 0 { + return fmt.Errorf("source has no non-nil variant") + } + + return nil +} + // LoadSpec loads a spec from the given data. // env is a map of environment variables to use for shell-style expansion in the spec. func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { @@ -82,6 +127,10 @@ func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { } for name, src := range spec.Sources { + if err := src.validate(); err != nil { + return nil, fmt.Errorf("error validating source ref %q: %w", name, err) + } + if err := src.processArgs(args); err != nil { return nil, fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) } From 851331ce3e49ce79f9fb6d9c62c19140a20474b5 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 15 Dec 2023 13:09:12 -0500 Subject: [PATCH 10/39] Allow `Cmd` subfield on docker image sources only It may be necessary in the future to allow specification of commands on `Build` and `Context` source variants. For now, it is only possible for Docker images. Signed-off-by: Peter Engelbert --- load.go | 14 ++++++++------ source.go | 25 ++++++++++++------------- spec.go | 12 ++++++------ 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/load.go b/load.go index 33221364a..a82642470 100644 --- a/load.go +++ b/load.go @@ -134,8 +134,10 @@ func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { if err := src.processArgs(args); err != nil { return nil, fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) } - if err := src.Cmd.processBuildArgs(lex, args, name); err != nil { - return nil, fmt.Errorf("error performing shell expansion on source %q: %w", name, err) + if src.DockerImage != nil { + if err := src.DockerImage.Cmd.processBuildArgs(lex, args, name); err != nil { + return nil, fmt.Errorf("error performing shell expansion on source %q: %w", name, err) + } } spec.Sources[name] = src } @@ -207,7 +209,7 @@ func (s *BuildStep) processBuildArgs(lex *shell.Lex, args map[string]string, i i return nil } -func (c *SourceCommand) processBuildArgs(lex *shell.Lex, args map[string]string, name string) error { +func (c *Command) processBuildArgs(lex *shell.Lex, args map[string]string, name string) error { if c == nil { return nil } @@ -239,8 +241,8 @@ func (c *SourceCommand) processBuildArgs(lex *shell.Lex, args map[string]string, func (s Spec) Validate() error { for name, src := range s.Sources { - if src.Cmd != nil { - for p, cfg := range src.Cmd.CacheDirs { + if src.DockerImage != nil && src.DockerImage.Cmd != nil { + for p, cfg := range src.DockerImage.Cmd.CacheDirs { if _, err := sharingMode(cfg.Mode); err != nil { return errors.Wrapf(err, "invalid sharing mode for source %q with cache mount at path %q", name, p) } @@ -295,7 +297,7 @@ func (c *CheckOutput) processBuildArgs(lex *shell.Lex, args map[string]string) e } func (c *TestSpec) processBuildArgs(lex *shell.Lex, args map[string]string, name string) error { - if err := c.SourceCommand.processBuildArgs(lex, args, name); err != nil { + if err := c.Command.processBuildArgs(lex, args, name); err != nil { return err } diff --git a/source.go b/source.go index 903227794..7638bdef1 100644 --- a/source.go +++ b/source.go @@ -35,7 +35,7 @@ func shArgs(cmd string) llb.RunOption { } // must not be called with a nil cmd pointer -func generateSourceFromImage(s *Spec, name string, st llb.State, cmd *SourceCommand, sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.ExecState, error) { +func generateSourceFromImage(s *Spec, name string, st llb.State, cmd *Command, sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.ExecState, error) { var zero llb.ExecState if len(cmd.Steps) == 0 { return zero, fmt.Errorf("no steps defined for image source") @@ -136,11 +136,11 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter img := src.DockerImage st := llb.Image(img.Ref, llb.WithMetaResolver(sOpt.Resolver), withConstraints(opts)) - if src.Cmd == nil { + if img.Cmd == nil { return st, nil } - eSt, err := generateSourceFromImage(s, name, st, src.Cmd, sOpt, opts...) + eSt, err := generateSourceFromImage(s, name, st, img.Cmd, sOpt, opts...) if err != nil { return llb.Scratch(), err } @@ -195,7 +195,6 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter Path: src.Path, Includes: src.Includes, Excludes: src.Excludes, - Cmd: src.Cmd, } st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) if err != nil { @@ -313,7 +312,7 @@ func (s Source) Doc() (io.Reader, error) { } case s.DockerImage != nil: img := s.DockerImage - if s.Cmd == nil { + if img.Cmd == nil { fmt.Fprintln(b, "Generated from a docker image:") fmt.Fprintln(b, " Image:", img.Ref) if s.Path != "" { @@ -325,19 +324,19 @@ func (s Source) Doc() (io.Reader, error) { if s.Path != "" { fmt.Fprintln(b, " Extraced path:", s.Path) } - if len(s.Cmd.Env) > 0 { + if len(img.Cmd.Env) > 0 { fmt.Fprintln(b, " With the following environment variables set for all commands:") - sorted := SortMapKeys(s.Cmd.Env) + sorted := SortMapKeys(img.Cmd.Env) for _, k := range sorted { - fmt.Fprintf(b, " %s=%s\n", k, s.Cmd.Env[k]) + fmt.Fprintf(b, " %s=%s\n", k, img.Cmd.Env[k]) } } - if s.Cmd.Dir != "" { - fmt.Fprintln(b, " Working Directory:", s.Cmd.Dir) + if img.Cmd.Dir != "" { + fmt.Fprintln(b, " Working Directory:", img.Cmd.Dir) } fmt.Fprintln(b, " Command(s):") - for _, step := range s.Cmd.Steps { + for _, step := range img.Cmd.Steps { fmt.Fprintf(b, " %s\n", step.Command) if len(step.Env) > 0 { fmt.Fprintln(b, " With the following environment variables set for this command:") @@ -347,9 +346,9 @@ func (s Source) Doc() (io.Reader, error) { } } } - if len(s.Cmd.Mounts) > 0 { + if len(img.Cmd.Mounts) > 0 { fmt.Fprintln(b, " With the following items mounted:") - for _, src := range s.Cmd.Mounts { + for _, src := range img.Cmd.Mounts { sub, err := src.Spec.Doc() if err != nil { return nil, err diff --git a/spec.go b/spec.go index dced06c0f..dbc53ffa0 100644 --- a/spec.go +++ b/spec.go @@ -180,7 +180,8 @@ type ImageConfig struct { } type SourceDockerImage struct { - Ref string `yaml:"ref" json:"ref"` + Ref string `yaml:"ref" json:"ref"` + Cmd *Command `yaml:"cmd,omitempty" json:"cmd,omitempty"` } type SourceGit struct { @@ -219,8 +220,8 @@ type SourceBuild struct { Inline string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` } -// SourceCommand is used to execute a command to generate a source from a docker image. -type SourceCommand struct { +// Command is used to execute a command to generate a source from a docker image. +type Command struct { // Dir is the working directory to run the command in. Dir string `yaml:"dir,omitempty" json:"dir,omitempty"` @@ -256,7 +257,6 @@ type Source struct { Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` Build *SourceBuild `yaml:"build,omitempty" json:"build,omitempty"` Local *SourceLocal `yaml:"local,omitempty" json:"local,omitempty"` - Cmd *SourceCommand `yaml:"cmd,omitempty" json:"cmd,omitempty"` // === End Source Variants === // Path is the path to the source after fetching it based on the identifier. @@ -387,8 +387,8 @@ type Target struct { type TestSpec struct { // Name is the name of the test // This will be used to output the test results - Name string `yaml:"name" json:"name" jsonschema:"required"` - SourceCommand `yaml:",inline"` + Name string `yaml:"name" json:"name" jsonschema:"required"` + Command `yaml:",inline"` // Steps is the list of commands to run to test the package. Steps []TestStep `yaml:"steps" json:"steps" jsonschema:"required"` // Files is the list of files to check after running the steps. From 320f1467092d982fac277d5c557be3dc72734a46 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 15 Dec 2023 13:12:01 -0500 Subject: [PATCH 11/39] Remove dead code Signed-off-by: Peter Engelbert --- source.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/source.go b/source.go index 7638bdef1..9d21c073f 100644 --- a/source.go +++ b/source.go @@ -13,13 +13,6 @@ import ( "github.com/moby/buildkit/util/gitutil" ) -const ( - // Custom source type to generate output from a command. - sourceTypeContext = "context" - sourceTypeBuild = "build" - sourceTypeSource = "source" -) - type LLBGetter func(sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) type ForwarderFunc func(llb.State, *SourceBuild) (llb.State, error) @@ -250,11 +243,6 @@ func SourceIsDir(src Source) (bool, error) { } } -func isGitRef(ref string) bool { - _, err := gitutil.ParseGitRef(ref) - return err == nil -} - // Doc returns the details of how the source was created. // This should be included, where applicable, in build in build specs (such as RPM spec files) // so that others can reproduce the build. From e649cad052ee8055a73c7322d3151b86f2386ffc Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 15 Dec 2023 14:41:46 -0500 Subject: [PATCH 12/39] Remove JSON schema extension The format has changed, and this check is no longer correct. This commit also re-runs the `go generate` command and commits the changes. Signed-off-by: Peter Engelbert --- docs/spec.schema.json | 171 +++++++++++++++++++++++++++++------------- spec.go | 18 ----- 2 files changed, 119 insertions(+), 70 deletions(-) diff --git a/docs/spec.schema.json b/docs/spec.schema.json index a645977db..05edea3a1 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -63,35 +63,6 @@ "type": "object", "description": "Artifacts describes all the artifacts to include in the package." }, - "BuildSpec": { - "properties": { - "target": { - "type": "string", - "description": "Target specifies the build target to use.\nIf unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default)." - }, - "args": { - "additionalProperties": { - "type": "string" - }, - "type": "object", - "description": "Args are the build args to pass to the build." - }, - "file": { - "type": "string", - "description": "File is the path to the build file in the build context\nIf not set the default is assumed by buildkit to be `Dockerfile` at the root of the context.\nThis is exclusive with [Inline]" - }, - "inline": { - "type": "string", - "description": "Inline is an inline build spec to use.\nThis can be used to specify a dockerfile instead of using one in the build context\nThis is exclusive with [File]", - "examples": [ - "FROM busybox\nRUN echo hello world" - ] - } - }, - "additionalProperties": false, - "type": "object", - "description": "BuildSpec is used to generate source from a build." - }, "BuildStep": { "properties": { "command": { @@ -211,7 +182,7 @@ "type": "object", "description": "CheckOutput is used to specify the exepcted output of a check, such as stdout/stderr or a file." }, - "CmdSpec": { + "Command": { "properties": { "dir": { "type": "string", @@ -251,7 +222,7 @@ "required": [ "steps" ], - "description": "CmdSpec is used to execute a command to generate a source from a docker image." + "description": "Command is used to execute a command to generate a source from a docker image." }, "FileCheckOutput": { "properties": { @@ -437,17 +408,24 @@ }, "Source": { "properties": { - "ref": { - "type": "string", - "pattern": "^((context|docker-image|git|http|https|source)://.+)|((context|build)://)$", - "description": "Ref is a unique identifier for the source.\nExample: \"docker-image://busybox:latest\", \"https://github.com/moby/buildkit.git#master\", \"context://\n\nWhen a source is specified as part of a [CmdSpec], it may also be used to reference a top-level source.", - "examples": [ - "docker-image://busybox:latest", - "https://github.com/moby/buildkit.git#master", - "build://", - "context://", - "context://some/path/in/build/context" - ] + "image": { + "$ref": "#/$defs/SourceDockerImage", + "description": "This is an embedded union representing all of the possible source types.\nOnly one should be non-nil at any given time. It is considered an error\ncondition if more than one is non-nil, or if all are nil.\n\n=== Begin Source Variants ===" + }, + "git": { + "$ref": "#/$defs/SourceGit" + }, + "https": { + "$ref": "#/$defs/SourceHTTPS" + }, + "context": { + "$ref": "#/$defs/SourceContext" + }, + "build": { + "$ref": "#/$defs/SourceBuild" + }, + "local": { + "$ref": "#/$defs/SourceLocal" }, "path": { "type": "string", @@ -466,26 +444,115 @@ }, "type": "array", "description": "Excludes is a list of paths underneath `Path` to exclude, everything else is included" + } + }, + "additionalProperties": false, + "type": "object", + "description": "Source defines a source to be used in the build." + }, + "SourceBuild": { + "properties": { + "name": { + "type": "string" }, - "keep_git_dir": { - "type": "boolean", - "description": "KeepGitDir is used to keep the .git directory after fetching the source for git references." + "target": { + "type": "string", + "description": "Target specifies the build target to use.\nIf unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default)." }, - "cmd": { - "$ref": "#/$defs/CmdSpec", - "description": "Cmd is used to generate the source from a command.\nThis can be used when Ref is \"docker-image://\"\nIf ref is \"cmd://\", this is required." + "args": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Args are the build args to pass to the build." }, - "build": { - "$ref": "#/$defs/BuildSpec", - "description": "Build is used to generate source from a build.\nThis is used when [Ref]` is \"build://\"\nThe context for the build is assumed too be specified in after `build://` in the ref, e.g. `build://https://github.com/moby/buildkit.git#master`\nWhen nothing is specified after `build://`, the context is assumed to be the current build context." + "file": { + "type": "string", + "description": "File is the path to the build file in the build context\nIf not set the default is assumed by buildkit to be `Dockerfile` at the root of the context.\nThis is exclusive with [Inline]" + }, + "inline": { + "type": "string", + "description": "Inline is an inline build spec to use.\nThis can be used to specify a dockerfile instead of using one in the build context\nThis is exclusive with [File]", + "examples": [ + "FROM busybox\nRUN echo hello world" + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "name" + ], + "description": "i.e." + }, + "SourceContext": { + "properties": { + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "name" + ] + }, + "SourceDockerImage": { + "properties": { + "ref": { + "type": "string" + }, + "cmd": { + "$ref": "#/$defs/Command" } }, "additionalProperties": false, "type": "object", "required": [ "ref" + ] + }, + "SourceGit": { + "properties": { + "url": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "keepGitDir": { + "type": "boolean" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "url", + "commit", + "keepGitDir" + ] + }, + "SourceHTTPS": { + "properties": { + "url": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "url" ], - "description": "Source defines a source to be used in the build." + "description": "No longer supports `.git` URLs as git repos." + }, + "SourceLocal": { + "properties": { + "path": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" }, "SourceMount": { "properties": { diff --git a/spec.go b/spec.go index dbc53ffa0..06110668e 100644 --- a/spec.go +++ b/spec.go @@ -7,8 +7,6 @@ import ( "regexp" "strings" "time" - - "github.com/invopop/jsonschema" ) // Spec is the specification for a package build. @@ -269,22 +267,6 @@ type Source struct { Excludes []string `yaml:"excludes,omitempty" json:"excludes,omitempty"` } -func (Source) JSONSchemaExtend(schema *jsonschema.Schema) { - s, ok := schema.Properties.Get("ref") - if !ok { - panic("ref property not found") - } - s.Pattern = `^((context|docker-image|git|http|https|source)://.+)|((context|build)://)$` - - s.Examples = []interface{}{ - "docker-image://busybox:latest", - "https://github.com/moby/buildkit.git#master", - "build://", - "context://", - "context://some/path/in/build/context", - } -} - // PackageDependencies is a list of dependencies for a package. // This will be included in the package metadata so that the package manager can install the dependencies. // It also includes build-time dedendencies, which we'll install before running any build steps. From 30aa0b806e1b4a7e85dec2aed9466380bdaba112 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 18 Dec 2023 12:37:34 -0500 Subject: [PATCH 13/39] Match some tests fixtures to new spec format Note: Only some of the tests will run properly. The logic for `SourceBuild` needs to be re-tooled in order to run one of the tests properly. I will come back later to edit the rest of the test fixtures. Signed-off-by: Peter Engelbert --- test/fixtures/cmd-src-ref.yml | 24 +++++++++++++----------- test/fixtures/frontend.yml | 26 +++++++++++++------------- test/fixtures/http-src.yml | 8 +++++--- test/fixtures/local-context.yml | 15 ++++++++++----- 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/test/fixtures/cmd-src-ref.yml b/test/fixtures/cmd-src-ref.yml index c1e38a48e..e70b1a2ac 100644 --- a/test/fixtures/cmd-src-ref.yml +++ b/test/fixtures/cmd-src-ref.yml @@ -14,17 +14,19 @@ packager: Microsoft license: Apache 2.0 sources: test: - ref: docker-image://busybox:latest path: /bar - cmd: - mounts: - - dest: "/foo" - spec: - ref: context:// - steps: - - env: - FILE_TO_IMPORT: ${file_to_import} - command: set -e; mkdir -p /bar; cp "/foo/${FILE_TO_IMPORT}" "/bar/${FILE_TO_IMPORT}" + image: + ref: busybox:latest + cmd: + mounts: + - dest: "/foo" + spec: + context: + name: "" + steps: + - env: + FILE_TO_IMPORT: ${file_to_import} + command: set -e; mkdir -p /bar; cp "/foo/${FILE_TO_IMPORT}" "/bar/${FILE_TO_IMPORT}" build: @@ -32,4 +34,4 @@ build: FILE_TO_IMPORT: ${file_to_import} steps: - command: | - [ -f "test/${FILE_TO_IMPORT}" ] \ No newline at end of file + [ -f "test/${FILE_TO_IMPORT}" ] diff --git a/test/fixtures/frontend.yml b/test/fixtures/frontend.yml index 50ae621df..a37971fe0 100644 --- a/test/fixtures/frontend.yml +++ b/test/fixtures/frontend.yml @@ -10,21 +10,21 @@ vendor: Microsoft packager: Microsoft license: Apache 2.0 sources: - src: - ref: context:// gomodcache: - ref: docker-image://mcr.microsoft.com/oss/go/microsoft/golang:1.21 path: /build/gomodcache - cmd: - dir: /build/src - mounts: - - dest: /build/src - spec: - ref: source://src - steps: - - command: go mod download - env: - GOMODCACHE: /build/gomodcache + image: + ref: mcr.microsoft.com/oss/go/microsoft/golang:1.21 + cmd: + dir: /build/src + mounts: + - dest: /build/src + spec: + context: + name: "" + steps: + - command: go mod download + env: + GOMODCACHE: /build/gomodcache dependencies: build: diff --git a/test/fixtures/http-src.yml b/test/fixtures/http-src.yml index 87e46fa8a..1f7a1c6d1 100644 --- a/test/fixtures/http-src.yml +++ b/test/fixtures/http-src.yml @@ -12,12 +12,14 @@ license: Apache 2.0 sources: readme: - ref: https://raw.githubusercontent.com/moby/buildkit/2677a22857c917168730fe69ad617a50e0d85202/README.md + https: + url: https://raw.githubusercontent.com/moby/buildkit/2677a22857c917168730fe69ad617a50e0d85202/README.md SOMEOTHERNAME.md: - ref: https://raw.githubusercontent.com/moby/buildkit/2677a22857c917168730fe69ad617a50e0d85202/README.md + https: + url: https://raw.githubusercontent.com/moby/buildkit/2677a22857c917168730fe69ad617a50e0d85202/README.md build: steps: - command: | [ -f readme ] - [ -f SOMEOTHERNAME.md ] \ No newline at end of file + [ -f SOMEOTHERNAME.md ] diff --git a/test/fixtures/local-context.yml b/test/fixtures/local-context.yml index 3e5ea60a2..0d15f5822 100644 --- a/test/fixtures/local-context.yml +++ b/test/fixtures/local-context.yml @@ -11,21 +11,26 @@ packager: Microsoft license: Apache 2.0 sources: src: - ref: context:// + context: + name: "" src2: - ref: context://go.mod + context: + name: "go.mod" src3: - ref: context:// + context: + name: "" includes: - go.mod - go.sum src4: - ref: context:// + context: + name: "" excludes: - go.mod - go.sum src5: - ref: context:// + context: + name: "" includes: - go.* excludes: From 5a723c78feae3cd919bb8bd8d3328ccf0805549e Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 18 Dec 2023 13:49:51 -0500 Subject: [PATCH 14/39] Retool `SourceBuild` `SourceBuild` needs a base state of some sort, in which the specified Dockerfile will be executed. This implementation allows the base state to be made up of a local path, a build context, or another source. This may not be final, and it's likely that there are mistakes in here. Signed-off-by: Peter Engelbert --- frontend/gateway.go | 16 ++++++------- frontend/rpm/handle_sources.go | 9 +++++++- load.go | 9 +++++++- source.go | 42 ++++++++++++++++++++++------------ spec.go | 33 ++++++++++++++++++-------- 5 files changed, 74 insertions(+), 35 deletions(-) diff --git a/frontend/gateway.go b/frontend/gateway.go index 1b4ac27d5..717d08bce 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -25,14 +25,14 @@ const ( dalecSubrequstForwardBuild = "dalec.forward.build" ) -func getDockerfile(ctx context.Context, client gwclient.Client, spec *dalec.SourceBuild, defPb *pb.Definition) ([]byte, error) { - if spec.Inline != "" { - return []byte(spec.Inline), nil - } - +func getDockerfile(ctx context.Context, client gwclient.Client, build *dalec.SourceBuild, defPb *pb.Definition) ([]byte, error) { dockerfilePath := dockerui.DefaultDockerfileName - if spec.File != "" { - dockerfilePath = spec.File + + switch { + case build.Inline != nil: + return []byte(*build.Inline), nil + case build.DockerFile != nil: + dockerfilePath = *build.DockerFile } // First we need to read the dockerfile to determine what frontend to forward to @@ -65,7 +65,7 @@ func ForwarderFromClient(ctx context.Context, client gwclient.Client) dalec.Forw spec = &dalec.SourceBuild{} } - if spec.File != "" && spec.Inline != "" { + if spec.DockerFile != nil && spec.Inline != nil { return llb.Scratch(), fmt.Errorf("cannot specify both file and inline for build spec") } diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index 565406ca5..2189b95c6 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -99,7 +99,14 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err case src.Context != nil: s = src.Context.Name case src.Build != nil: - s = src.Build.Name + switch { + case src.Build.Context != nil: + s = src.Build.Context.Name + case src.Build.Local != nil: + s = src.Build.Local.Path + case src.Build.Source != nil: + s = src.Build.Source.Name + } default: } diff --git a/load.go b/load.go index a82642470..a8b529528 100644 --- a/load.go +++ b/load.go @@ -39,7 +39,14 @@ func (s *Source) processArgs(args map[string]string) error { *sub = "context" } case s.Build != nil: - sub = &s.Build.Name + switch { + case s.Build.Context != nil: + sub = &s.Build.Context.Name + case s.Build.Local != nil: + sub = &s.Build.Local.Path + case s.Build.Source != nil: + sub = &s.Build.Source.Name + } case s.Local != nil: sub = &s.Local.Path default: diff --git a/source.go b/source.go index 9d21c073f..d969431bd 100644 --- a/source.go +++ b/source.go @@ -180,22 +180,33 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter var err error build := src.Build var st llb.State - if build.Name == "" { - st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) - } else { - src2 := Source{ - Build: &SourceBuild{Name: build.Name}, - Path: src.Path, - Includes: src.Includes, - Excludes: src.Excludes, + switch { + case build.Context != nil: + if build.Context.Name == "" { + st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) + } else { + src2 := Source{ + Context: &SourceContext{Name: build.Context.Name}, + Path: src.Path, + Includes: src.Includes, + Excludes: src.Excludes, + } + st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) + if err != nil { + return llb.Scratch(), err + } } + case build.Local != nil: + st = llb.Local(build.Local.Path, withConstraints(opts)) + case build.Source != nil: + src2 := s.Sources[name] st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) if err != nil { return llb.Scratch(), err } } - return sOpt.Forward(st, src.Build) + return sOpt.Forward(st, build) default: return llb.Scratch(), fmt.Errorf("No source variant found") } @@ -257,7 +268,7 @@ func (s Source) Doc() (io.Reader, error) { build := s.Build fmt.Fprintln(b, "Generated from a docker build:") fmt.Fprintln(b, " Docker Build Target:", s.Build.Target) - fmt.Fprintln(b, " Docker Build Ref:", build.Name) + fmt.Fprintln(b, " Docker Build Ref:", build.Context) if len(s.Build.Args) > 0 { sorted := SortMapKeys(s.Build.Args) @@ -267,20 +278,21 @@ func (s Source) Doc() (io.Reader, error) { } } - if s.Build.Inline != "" { + switch { + case s.Build.Inline != nil: fmt.Fprintln(b, " Dockerfile:") - scanner := bufio.NewScanner(strings.NewReader(s.Build.Inline)) + scanner := bufio.NewScanner(strings.NewReader(*s.Build.Inline)) for scanner.Scan() { fmt.Fprintf(b, " %s\n", scanner.Text()) } if scanner.Err() != nil { return nil, scanner.Err() } - } else { + case s.Build.DockerFile != nil: p := "Dockerfile" - if s.Build.File != "" { - p = s.Build.File + if s.Build.DockerFile != nil { + p = *s.Build.DockerFile } fmt.Fprintln(b, " Dockerfile path in context:", p) } diff --git a/spec.go b/spec.go index 06110668e..ded4f71a4 100644 --- a/spec.go +++ b/spec.go @@ -201,21 +201,34 @@ type SourceContext struct { // i.e. just rename `BuildSpec` to `SourceBuild` // SourceBuild is used to generate source from a build. type SourceBuild struct { - Name string `yaml:"name" json:"name"` - // Target specifies the build target to use. - // If unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default). - Target string `yaml:"target,omitempty" json:"target,omitempty"` - // Args are the build args to pass to the build. - Args map[string]string `yaml:"args,omitempty" json:"args,omitempty"` - // File is the path to the build file in the build context + // An embedded union representing the different build contexts + // === Begin SourceBuild Base Variants === // + // A build context provided by the user via a cli invocation + Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` + // A local directory + Local *SourceLocal `yaml:"local,omitempty" json:"local,omitempty"` + // A reference to another source + Source *SourceContext `yaml:"source,omitempty" json:"source,omitempty"` + // === End SourceBuild Base Variants === // + + // An embedded union representing the different ways of running a build + // === Begin SourceBuild Action Variants === + // DockerFile is the path to the build file in the build context // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. // This is exclusive with [Inline] - File string `yaml:"file,omitempty" json:"file,omitempty"` - + DockerFile *string `yaml:"file,omitempty" json:"file,omitempty"` // Inline is an inline build spec to use. // This can be used to specify a dockerfile instead of using one in the build context // This is exclusive with [File] - Inline string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` + Inline *string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` + // === End SourceBuild Action Variants === + + // Target specifies the build target to use. + // If unset, the default target is determined by the frontend implementation + // (e.g. the dockerfile frontend uses the last build stage as the default). + Target string `yaml:"target,omitempty" json:"target,omitempty"` + // Args are the build args to pass to the build. + Args map[string]string `yaml:"args,omitempty" json:"args,omitempty"` } // Command is used to execute a command to generate a source from a docker image. From 19aefe669c75ec2f1ca0fcf871f1a74d288964b4 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 18 Dec 2023 13:52:55 -0500 Subject: [PATCH 15/39] Fix more yaml fixtures The tests are still not completing successfully, so more troubleshooting is needed. Signed-off-by: Peter Engelbert --- test/fixtures/moby-runc.yml | 3 ++- test/fixtures/nested.yml | 5 +++-- test/fixtures/test-framework.yml | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/fixtures/moby-runc.yml b/test/fixtures/moby-runc.yml index 7995c6215..8f866feaa 100644 --- a/test/fixtures/moby-runc.yml +++ b/test/fixtures/moby-runc.yml @@ -61,7 +61,8 @@ provides: - runc sources: src: - ref: https://github.com/opencontainers/runc.git#${RUNC_COMMIT} + https: + url: https://github.com/opencontainers/runc.git#${RUNC_COMMIT} build: env: diff --git a/test/fixtures/nested.yml b/test/fixtures/nested.yml index c2d1ac109..a2007c53e 100644 --- a/test/fixtures/nested.yml +++ b/test/fixtures/nested.yml @@ -11,9 +11,10 @@ packager: Microsoft license: Apache 2.0 noarch: true sources: - src: - ref: build://context://test/fixtures/nested + src2: build: + local: + path: test/fixtures/nested inline: | # syntax=docker/dockerfile:1.5 FROM scratch diff --git a/test/fixtures/test-framework.yml b/test/fixtures/test-framework.yml index a7f0851b7..1f9e09f45 100644 --- a/test/fixtures/test-framework.yml +++ b/test/fixtures/test-framework.yml @@ -20,17 +20,17 @@ tests: steps: - command: cat stdin: "Hello, world!" - stdout: + stdout: equals: "Hello, world!" - name: Check multi-argument command steps: - command: echo "Hello, world!" - stdout: + stdout: equals: "Hello, world!\n" - command: /bin/sh -c 'echo "Hello, world!" >&2' - stderr: + stderr: equals: "Hello, world!\n" From b3570a2da3ef3fbcb65e833792584d82c372a426 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 18 Dec 2023 16:25:51 -0500 Subject: [PATCH 16/39] All tests but one are passing Signed-off-by: Peter Engelbert --- source.go | 21 +++++++++++++-------- test/fixtures/local-context.yml | 8 ++++---- test/fixtures/moby-runc.yml | 2 +- test/fixtures/nested.yml | 10 +++++++--- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/source.go b/source.go index d969431bd..cb9494a49 100644 --- a/source.go +++ b/source.go @@ -163,6 +163,11 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter return llb.HTTP(https.URL, opts...), nil case src.Context != nil: srcCtx := src.Context + ctxName := srcCtx.Name + if ctxName == "" { + ctxName = "." + } + st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) if err != nil { return llb.Scratch(), err @@ -185,21 +190,21 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter if build.Context.Name == "" { st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) } else { - src2 := Source{ - Context: &SourceContext{Name: build.Context.Name}, - Path: src.Path, - Includes: src.Includes, - Excludes: src.Excludes, - } - st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) + st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) if err != nil { return llb.Scratch(), err } + + includeExcludeHandled = true + if src.Path == "" && build.Context.Name != "" { + src.Path = build.Context.Name + } + return *st, nil } case build.Local != nil: st = llb.Local(build.Local.Path, withConstraints(opts)) case build.Source != nil: - src2 := s.Sources[name] + src2 := s.Sources[build.Source.Name] st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) if err != nil { return llb.Scratch(), err diff --git a/test/fixtures/local-context.yml b/test/fixtures/local-context.yml index 0d15f5822..58f3b00e7 100644 --- a/test/fixtures/local-context.yml +++ b/test/fixtures/local-context.yml @@ -12,25 +12,25 @@ license: Apache 2.0 sources: src: context: - name: "" + name: "." src2: context: name: "go.mod" src3: context: - name: "" + name: "." includes: - go.mod - go.sum src4: context: - name: "" + name: "." excludes: - go.mod - go.sum src5: context: - name: "" + name: "." includes: - go.* excludes: diff --git a/test/fixtures/moby-runc.yml b/test/fixtures/moby-runc.yml index 8f866feaa..ffda3e714 100644 --- a/test/fixtures/moby-runc.yml +++ b/test/fixtures/moby-runc.yml @@ -66,7 +66,7 @@ sources: build: env: - CGO_ENABLED: 1 + CGO_ENABLED: 1 GOGC: off GOFLAGS: -trimpath GOROOT: /usr/lib/golang diff --git a/test/fixtures/nested.yml b/test/fixtures/nested.yml index a2007c53e..267c2cdc6 100644 --- a/test/fixtures/nested.yml +++ b/test/fixtures/nested.yml @@ -11,10 +11,14 @@ packager: Microsoft license: Apache 2.0 noarch: true sources: - src2: + foo: + context: + name: test/fixtures/nested + # path: test/fixtures/nested + src: build: - local: - path: test/fixtures/nested + source: + name: foo inline: | # syntax=docker/dockerfile:1.5 FROM scratch From b33b3aff2fbf11b53f876246077d867249289e43 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 8 Jan 2024 11:20:27 -0500 Subject: [PATCH 17/39] Redesign `SourceBuild` struct After discussion with Brian, this is the correct struct layout for this. The context string is a path which will be used as the `.` context for the dockerfile build. Signed-off-by: Peter Engelbert --- spec.go | 19 +++++-------------- test/fixtures/nested.yml | 7 +------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/spec.go b/spec.go index ded4f71a4..a684395dc 100644 --- a/spec.go +++ b/spec.go @@ -198,21 +198,12 @@ type SourceContext struct { Name string `yaml:"name" json:"name"` } -// i.e. just rename `BuildSpec` to `SourceBuild` -// SourceBuild is used to generate source from a build. +// SourceBuild is used to generate source from a DockerFile build, either +// inline or from a local file. type SourceBuild struct { - // An embedded union representing the different build contexts - // === Begin SourceBuild Base Variants === // - // A build context provided by the user via a cli invocation - Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` - // A local directory - Local *SourceLocal `yaml:"local,omitempty" json:"local,omitempty"` - // A reference to another source - Source *SourceContext `yaml:"source,omitempty" json:"source,omitempty"` - // === End SourceBuild Base Variants === // - - // An embedded union representing the different ways of running a build - // === Begin SourceBuild Action Variants === + // The local path to use as the context for the Dockerfile build + Context string `yaml:"context,omitempty" json:"context,omitempty"` + // DockerFile is the path to the build file in the build context // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. // This is exclusive with [Inline] diff --git a/test/fixtures/nested.yml b/test/fixtures/nested.yml index 267c2cdc6..a41030240 100644 --- a/test/fixtures/nested.yml +++ b/test/fixtures/nested.yml @@ -11,14 +11,9 @@ packager: Microsoft license: Apache 2.0 noarch: true sources: - foo: - context: - name: test/fixtures/nested - # path: test/fixtures/nested src: build: - source: - name: foo + context: test/fixtures/nested inline: | # syntax=docker/dockerfile:1.5 FROM scratch From 7b6a8756f672069d4b29dcf3d6774287b9fed52e Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Mon, 8 Jan 2024 12:52:30 -0500 Subject: [PATCH 18/39] Save progress Signed-off-by: Peter Engelbert --- source.go | 142 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 61 deletions(-) diff --git a/source.go b/source.go index cb9494a49..bd27ed524 100644 --- a/source.go +++ b/source.go @@ -77,6 +77,58 @@ func Source2LLBGetter(s *Spec, src Source, name string) LLBGetter { return source2LLBGetter(s, src, name, false) } +func needsFilter(o *filterOpts) bool { + if o.source.Path != "" && !o.forMount && !o.pathHandled { + return true + } + if o.includeExcludeHandled { + return false + } + if len(o.source.Includes) > 0 || len(o.source.Excludes) > 0 { + return true + } + return false +} + +type filterOpts struct { + state llb.State + source Source + opts []llb.ConstraintsOpt + forMount bool + includeExcludeHandled bool + pathHandled bool + err error +} + +func handleFilter(o *filterOpts) (llb.State, error) { + if o.err != nil { + return o.state, o.err + } + + if !needsFilter(o) { + return o.state, nil + } + + srcPath := "/" + if !o.pathHandled { + srcPath = o.source.Path + } + + filtered := llb.Scratch().File( + llb.Copy( + o.state, + srcPath, + "/", + WithIncludes(o.source.Includes), + WithExcludes(o.source.Excludes), + WithDirContentsOnly(), + ), + withConstraints(o.opts), + ) + + return filtered, nil +} + func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter { return func(sOpt SourceOpts, opts ...llb.ConstraintsOpt) (ret llb.State, retErr error) { var ( @@ -85,42 +137,15 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter ) defer func() { - if retErr != nil { - return - } - needsFilter := func() bool { - if src.Path != "" && !forMount && !pathHandled { - return true - } - if includeExcludeHandled { - return false - } - if len(src.Includes) > 0 || len(src.Excludes) > 0 { - return true - } - return false - } - if !needsFilter() { - return - } - - srcPath := "/" - if !pathHandled { - srcPath = src.Path - } - - orig := ret - ret = llb.Scratch().File( - llb.Copy( - orig, - srcPath, - "/", - WithIncludes(src.Includes), - WithExcludes(src.Excludes), - WithDirContentsOnly(), - ), - withConstraints(opts), - ) + ret, retErr = handleFilter(&filterOpts{ + state: ret, + source: src, + opts: opts, + forMount: forMount, + includeExcludeHandled: includeExcludeHandled, + pathHandled: pathHandled, + err: retErr, + }) }() // sourceType, err := src.GetSourceKind() @@ -163,10 +188,6 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter return llb.HTTP(https.URL, opts...), nil case src.Context != nil: srcCtx := src.Context - ctxName := srcCtx.Name - if ctxName == "" { - ctxName = "." - } st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) if err != nil { @@ -182,33 +203,32 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter srcLocal := src.Local return llb.Local(srcLocal.Path, localIncludeExcludeMerge(&src)), nil case src.Build != nil: - var err error build := src.Build var st llb.State - switch { - case build.Context != nil: - if build.Context.Name == "" { - st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) - } else { - st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) - if err != nil { - return llb.Scratch(), err - } - includeExcludeHandled = true - if src.Path == "" && build.Context.Name != "" { - src.Path = build.Context.Name - } - return *st, nil - } - case build.Local != nil: - st = llb.Local(build.Local.Path, withConstraints(opts)) - case build.Source != nil: - src2 := s.Sources[build.Source.Name] - st, err = source2LLBGetter(s, src2, name, forMount)(sOpt, opts...) + if build.Context == "" { + st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) + } else { + ctxState, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) if err != nil { return llb.Scratch(), err } + cst := *ctxState + + if src.Path == "" && build.Context != "" { + src.Path = build.Context + } + + // This is necessary to have the specified context to be at the + // root of the state's fs. + st, _ = handleFilter(&filterOpts{ + state: cst, + source: src, + opts: opts, + forMount: forMount, + includeExcludeHandled: false, + pathHandled: false, + }) } return sOpt.Forward(st, build) From b84de6f119d8b7d12d72a18f1f61920b0b119b16 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 09:54:34 -0500 Subject: [PATCH 19/39] save Signed-off-by: Peter Engelbert --- frontend/rpm/handle_sources.go | 11 +++-------- load.go | 9 +-------- source.go | 15 ++++++++------- spec.go | 2 +- 4 files changed, 13 insertions(+), 24 deletions(-) diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index 2189b95c6..5f73e563c 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -99,14 +99,9 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err case src.Context != nil: s = src.Context.Name case src.Build != nil: - switch { - case src.Build.Context != nil: - s = src.Build.Context.Name - case src.Build.Local != nil: - s = src.Build.Local.Path - case src.Build.Source != nil: - s = src.Build.Source.Name - } + s = src.Build.Context + case src.Local != nil: + s = src.Local.Path default: } diff --git a/load.go b/load.go index a8b529528..288a5c03d 100644 --- a/load.go +++ b/load.go @@ -39,14 +39,7 @@ func (s *Source) processArgs(args map[string]string) error { *sub = "context" } case s.Build != nil: - switch { - case s.Build.Context != nil: - sub = &s.Build.Context.Name - case s.Build.Local != nil: - sub = &s.Build.Local.Path - case s.Build.Source != nil: - sub = &s.Build.Source.Name - } + sub = &s.Build.Context case s.Local != nil: sub = &s.Local.Path default: diff --git a/source.go b/source.go index bd27ed524..cc2790238 100644 --- a/source.go +++ b/source.go @@ -215,18 +215,19 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } cst := *ctxState - if src.Path == "" && build.Context != "" { - src.Path = build.Context + src2 := src + if src2.Path == "" && build.Context != "" { + src2.Path = build.Context } // This is necessary to have the specified context to be at the // root of the state's fs. st, _ = handleFilter(&filterOpts{ state: cst, - source: src, + source: src2, opts: opts, forMount: forMount, - includeExcludeHandled: false, + includeExcludeHandled: true, pathHandled: false, }) } @@ -270,7 +271,8 @@ func SourceIsDir(src Source) (bool, error) { case src.DockerImage != nil, src.Git != nil, src.Build != nil, - src.Context != nil: + src.Context != nil, + src.Local != nil: return true, nil case src.HTTPS != nil: return false, nil @@ -426,14 +428,13 @@ func PatchSources(worker llb.State, spec *Spec, sourceToState map[string]llb.Sta sorted := SortMapKeys(spec.Sources) for _, sourceName := range sorted { - src := spec.Sources[sourceName] sourceState := states[sourceName] patches, patchesExist := spec.Patches[sourceName] if !patchesExist { continue } - pg := llb.ProgressGroup(pgID, "Patch spec source: "+sourceName+" "+src.Ref, false) + pg := llb.ProgressGroup(pgID, "Patch spec source: "+sourceName+" ", false) states[sourceName] = patchSource(worker, sourceState, states, patches, pg, withConstraints(opts)) } diff --git a/spec.go b/spec.go index a684395dc..53b3f55bb 100644 --- a/spec.go +++ b/spec.go @@ -207,7 +207,7 @@ type SourceBuild struct { // DockerFile is the path to the build file in the build context // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. // This is exclusive with [Inline] - DockerFile *string `yaml:"file,omitempty" json:"file,omitempty"` + DockerFile *string `yaml:"dockerfile,omitempty" json:"dockerfile,omitempty"` // Inline is an inline build spec to use. // This can be used to specify a dockerfile instead of using one in the build context // This is exclusive with [File] From 1612df8e17ce002babe0cd1559d5af72ee49dde9 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 10:41:51 -0500 Subject: [PATCH 20/39] Fix local source Signed-off-by: Peter Engelbert --- df.yml | 0 source.go | 8 +++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 df.yml diff --git a/df.yml b/df.yml new file mode 100644 index 000000000..e69de29bb diff --git a/source.go b/source.go index cc2790238..244dbb0cc 100644 --- a/source.go +++ b/source.go @@ -201,7 +201,13 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter return *st, nil case src.Local != nil: srcLocal := src.Local - return llb.Local(srcLocal.Path, localIncludeExcludeMerge(&src)), nil + + includeExcludeHandled = true + if src.Path == "" && srcLocal.Path != "" { + src.Path = srcLocal.Path + } + + return llb.Local(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)), nil case src.Build != nil: build := src.Build var st llb.State From fa813eddb41cc6e08c30f9955ce579d73b058ffe Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 10:45:18 -0500 Subject: [PATCH 21/39] Run `go generate` To pass the linter Signed-off-by: Peter Engelbert --- docs/spec.schema.json | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/docs/spec.schema.json b/docs/spec.schema.json index 05edea3a1..38b387378 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -452,23 +452,13 @@ }, "SourceBuild": { "properties": { - "name": { - "type": "string" - }, - "target": { + "context": { "type": "string", - "description": "Target specifies the build target to use.\nIf unset, the default target is determined by the frontend implementation (e.g. the dockerfile frontend uses the last build stage as the default)." + "description": "The local path to use as the context for the Dockerfile build" }, - "args": { - "additionalProperties": { - "type": "string" - }, - "type": "object", - "description": "Args are the build args to pass to the build." - }, - "file": { + "dockerfile": { "type": "string", - "description": "File is the path to the build file in the build context\nIf not set the default is assumed by buildkit to be `Dockerfile` at the root of the context.\nThis is exclusive with [Inline]" + "description": "DockerFile is the path to the build file in the build context\nIf not set the default is assumed by buildkit to be `Dockerfile` at the root of the context.\nThis is exclusive with [Inline]" }, "inline": { "type": "string", @@ -476,14 +466,22 @@ "examples": [ "FROM busybox\nRUN echo hello world" ] + }, + "target": { + "type": "string", + "description": "Target specifies the build target to use.\nIf unset, the default target is determined by the frontend implementation\n(e.g. the dockerfile frontend uses the last build stage as the default)." + }, + "args": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Args are the build args to pass to the build." } }, "additionalProperties": false, "type": "object", - "required": [ - "name" - ], - "description": "i.e." + "description": "SourceBuild is used to generate source from a DockerFile build, either inline or from a local file." }, "SourceContext": { "properties": { From 74db6e1f070329576567167cb8c1e0aa86874a31 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 10:46:21 -0500 Subject: [PATCH 22/39] Update comment for autogenerated docs Signed-off-by: Peter Engelbert --- docs/spec.schema.json | 2 +- spec.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/spec.schema.json b/docs/spec.schema.json index 38b387378..269e6834d 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -410,7 +410,7 @@ "properties": { "image": { "$ref": "#/$defs/SourceDockerImage", - "description": "This is an embedded union representing all of the possible source types.\nOnly one should be non-nil at any given time. It is considered an error\ncondition if more than one is non-nil, or if all are nil.\n\n=== Begin Source Variants ===" + "description": "This is an embedded union representing all of the possible source types.\nExactly one must be non-nil, with all other cases being errors.\n\n=== Begin Source Variants ===" }, "git": { "$ref": "#/$defs/SourceGit" diff --git a/spec.go b/spec.go index 53b3f55bb..fe6a4cff1 100644 --- a/spec.go +++ b/spec.go @@ -249,8 +249,7 @@ type SourceLocal struct { // A source can be a local directory, a git repositoryt, http(s) URL, etc. type Source struct { // This is an embedded union representing all of the possible source types. - // Only one should be non-nil at any given time. It is considered an error - // condition if more than one is non-nil, or if all are nil. + // Exactly one must be non-nil, with all other cases being errors. // // === Begin Source Variants === DockerImage *SourceDockerImage `yaml:"image,omitempty" json:"image,omitempty"` From ac0bffb51e411c6b2d5873a4c0ebb6eb85e11f9c Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 10:54:26 -0500 Subject: [PATCH 23/39] Update patching test to reflect new spec format Because of the patching PR. Signed-off-by: Peter Engelbert --- test/fixtures/git-patch.yml | 8 +++++--- test/fixtures/kubernetes-patch.yml | 15 +++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test/fixtures/git-patch.yml b/test/fixtures/git-patch.yml index be80bcb7d..928779b8e 100644 --- a/test/fixtures/git-patch.yml +++ b/test/fixtures/git-patch.yml @@ -14,9 +14,11 @@ dependencies: sources: src: - ref: https://github.com/moby/containerd.git + git: + url: https://github.com/moby/containerd.git patch1: - ref: https://raw.githubusercontent.com/Azure/moby-packaging/7092638d14708b389abf0937086508ceb2571404/moby-containerd/patches/service-execstart.patch + https: + url: https://raw.githubusercontent.com/Azure/moby-packaging/7092638d14708b389abf0937086508ceb2571404/moby-containerd/patches/service-execstart.patch build: steps: @@ -30,4 +32,4 @@ build: patches: src: - - source: patch1 \ No newline at end of file + - source: patch1 diff --git a/test/fixtures/kubernetes-patch.yml b/test/fixtures/kubernetes-patch.yml index 1e7ba3a4a..3681c8ee3 100644 --- a/test/fixtures/kubernetes-patch.yml +++ b/test/fixtures/kubernetes-patch.yml @@ -14,14 +14,17 @@ dependencies: sources: src: - ref: https://github.com/kubernetes/kubernetes.git#v1.28.0 - + git: + url: https://github.com/kubernetes/kubernetes.git#v1.28.0 patch_120129: - ref: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/120129.patch + https: + url: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/120129.patch patch_120134: - ref: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/120134.patch + https: + url: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/120134.patch patch_121882: - ref: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/121882.patch + https: + url: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/121882.patch build: @@ -44,4 +47,4 @@ patches: src: - source: patch_120129 - source: patch_120134 - - source: patch_121882 \ No newline at end of file + - source: patch_121882 From 107692321e05c26f4327491a14e13eaf82a1a5f7 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 11:43:26 -0500 Subject: [PATCH 24/39] Fix `frontend` test fixture We can no longer reference other sources. Instead, we can specify a `local` source with the current directory in order to build the frontend. Signed-off-by: Peter Engelbert --- test/fixtures/frontend.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/fixtures/frontend.yml b/test/fixtures/frontend.yml index a37971fe0..954626539 100644 --- a/test/fixtures/frontend.yml +++ b/test/fixtures/frontend.yml @@ -10,6 +10,9 @@ vendor: Microsoft packager: Microsoft license: Apache 2.0 sources: + src: + local: + path: . gomodcache: path: /build/gomodcache image: @@ -19,8 +22,8 @@ sources: mounts: - dest: /build/src spec: - context: - name: "" + local: + path: . steps: - command: go mod download env: From f068072e27dcd9cb89a2b0df5adfc7bf18595568 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 12:45:15 -0500 Subject: [PATCH 25/39] Fix runc test fixture Signed-off-by: Peter Engelbert --- test/fixtures/moby-runc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/moby-runc.yml b/test/fixtures/moby-runc.yml index ffda3e714..f6a23ce6f 100644 --- a/test/fixtures/moby-runc.yml +++ b/test/fixtures/moby-runc.yml @@ -61,7 +61,7 @@ provides: - runc sources: src: - https: + git: url: https://github.com/opencontainers/runc.git#${RUNC_COMMIT} build: From 991772ebf0050376c41234b06de9e4f7d18132f6 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Tue, 9 Jan 2024 16:47:33 -0500 Subject: [PATCH 26/39] Fix example spec files and accompanying docs Signed-off-by: Peter Engelbert --- docs/examples/go-md2man-1.yml | 6 +++-- docs/examples/go-md2man-2.yml | 31 ++++++++++++++------------ docs/intro.md | 41 ++++++++++++++++++++--------------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/docs/examples/go-md2man-1.yml b/docs/examples/go-md2man-1.yml index afd7a5a90..4d965e96d 100644 --- a/docs/examples/go-md2man-1.yml +++ b/docs/examples/go-md2man-1.yml @@ -1,3 +1,4 @@ +# syntax=ghcr.io/azure/dalec/frontend:latest name: go-md2man version: 2.0.3 packager: Dalec Example @@ -8,7 +9,8 @@ website: https://github.com/cpuguy83/go-md2man sources: src: - ref: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + git: + url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 dependencies: build: @@ -30,4 +32,4 @@ artifacts: image: entrypoint: go-md2man - cmd: --help \ No newline at end of file + cmd: --help diff --git a/docs/examples/go-md2man-2.yml b/docs/examples/go-md2man-2.yml index 11f0bdcde..8afffcfc3 100644 --- a/docs/examples/go-md2man-2.yml +++ b/docs/examples/go-md2man-2.yml @@ -9,22 +9,25 @@ website: https://github.com/cpuguy83/go-md2man sources: src: - ref: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + git: + url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 gomods: # This is required when the build environment does not allow network access. This downloads all the go modules. - ref: docker-image://mcr.microsoft.com/oss/go/microsoft/golang:1.21 path: /build/gomodcache # This is the path we will be extracing after running the command below. - cmd: - dir: /build/src - mounts: - # Mount the "src" source, specified above, so our command has access to it. - - dest: /build/src - spec: - ref: source://src - steps: - - command: go mod download - env: - # This variable controls where the go modules are downloaded to. - GOMODCACHE: /build/gomodcache + image: + ref: mcr.microsoft.com/oss/go/microsoft/golang:1.21 + cmd: + dir: /build/src + mounts: + # Mount a source (inline, under `spec`), so our command has access to it. + - dest: /build/src + spec: + git: + url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + steps: + - command: go mod download + env: + # This variable controls where the go modules are downloaded to. + GOMODCACHE: /build/gomodcache dependencies: build: diff --git a/docs/intro.md b/docs/intro.md index 39e82613d..06ab6f03b 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -138,7 +138,8 @@ website: https://github.com/cpuguy83/go-md2man sources: src: - ref: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + git: + url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 dependencies: build: @@ -166,8 +167,8 @@ image: In the `sources` section there is a single source called `src` that references the github repo at tag v2.0.3. The name `src` is arbitrary, however this is where the source will be checked out to in the build phase. You can add -multiple sources, in the build phase they will be checked out to the name you -give them. +multiple sources, and in the build phase they will be checked out to the name +you give them. One thing to note, in many build systems you will not have access to the internet while building the package, and indeed that is the case with the @@ -182,7 +183,7 @@ everything needed to build the package (aside from dependencies on other packages). Source packages can be published to a package repository and then another system can download the source package and build it. -In the case case of the above example, we need to include the go modules in the +In the case of the above example, we need to include the go modules in the list of sources. We'll accomplish this by add a source which will run `go mod download` in a docker image with the `src` source mounted and then extract the go modules from the resulting filesystem. @@ -190,6 +191,7 @@ go modules from the resulting filesystem. *Note*: See the full example from [examples/go-md2man.yml](examples/go-md2man-2.yml) ```yaml +# syntax=ghcr.io/azure/dalec/frontend:latest name: go-md2man version: 2.0.3 packager: Dalec Example @@ -200,22 +202,25 @@ website: https://github.com/cpuguy83/go-md2man sources: src: - ref: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + git: + url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 gomods: # This is required when the build environment does not allow network access. This downloads all the go modules. - ref: docker-image://mcr.microsoft.com/oss/go/microsoft/golang:1.21 path: /build/gomodcache # This is the path we will be extracing after running the command below. - cmd: - dir: /build/src - mounts: - # Mount the "src" source, specified above, so our command has access to it. - - dest: /build/src - spec: - ref: source://src - steps: - - command: go mod download - env: - # This variable controls where the go modules are downloaded to. - GOMODCACHE: /build/gomodcache + image: + ref: mcr.microsoft.com/oss/go/microsoft/golang:1.21 + cmd: + dir: /build/src + mounts: + # Mount a source (inline, under `spec`), so our command has access to it. + - dest: /build/src + spec: + git: + url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + steps: + - command: go mod download + env: + # This variable controls where the go modules are downloaded to. + GOMODCACHE: /build/gomodcache dependencies: build: From 8b795d1b3c9751859c0d5cd741af977cc0c9b688 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 10:18:00 -0500 Subject: [PATCH 27/39] Remove unused empty file Signed-off-by: Peter Engelbert --- df.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 df.yml diff --git a/df.yml b/df.yml deleted file mode 100644 index e69de29bb..000000000 From 007524ce2594d8aacd28820014d71114f362a765 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 10:18:11 -0500 Subject: [PATCH 28/39] Clean up validation function Signed-off-by: Peter Engelbert --- load.go | 52 +++++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/load.go b/load.go index 288a5c03d..d7061ef1e 100644 --- a/load.go +++ b/load.go @@ -58,49 +58,39 @@ func (s *Source) processArgs(args map[string]string) error { return nil } -func checkDuplicate[T any](c *int, p *T) error { - if c == nil { - return fmt.Errorf("validation function (check) called with nil pointer") - } - - if p != nil { - (*c)++ - } - - if *c > 1 { - return fmt.Errorf("more than one source variant defined") - } - - return nil -} - func (s *Source) validate() error { count := 0 - if err := checkDuplicate(&count, s.DockerImage); err != nil { - return err + if s.DockerImage != nil { + count++ } - if err := checkDuplicate(&count, s.Git); err != nil { - return err + if s.Git != nil { + count++ } - if err := checkDuplicate(&count, s.HTTPS); err != nil { - return err + if s.HTTPS != nil { + count++ } - if err := checkDuplicate(&count, s.Context); err != nil { - return err + if s.Context != nil { + count++ } - if err := checkDuplicate(&count, s.Build); err != nil { - return err + if s.Build != nil { + count++ } - if err := checkDuplicate(&count, s.Local); err != nil { - return err + if s.Local != nil { + count++ } - if count == 0 { - return fmt.Errorf("source has no non-nil variant") + var err error + switch count { + case 0: + err = fmt.Errorf("no non-nil source variant") + case 1: + err = nil + default: + err = fmt.Errorf("more than one source variant defined") } - return nil + return err } // LoadSpec loads a spec from the given data. From 44084e46769dd1cf8aa190e66dee464e17893c6a Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 10:19:13 -0500 Subject: [PATCH 29/39] Remove dead code Signed-off-by: Peter Engelbert --- source.go | 1 - 1 file changed, 1 deletion(-) diff --git a/source.go b/source.go index 244dbb0cc..fa6fe67f9 100644 --- a/source.go +++ b/source.go @@ -148,7 +148,6 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter }) }() - // sourceType, err := src.GetSourceKind() switch { case src.DockerImage != nil: img := src.DockerImage From 9554d2b41ba7459d0d18f856108792a1a81f5174 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 12:08:46 -0500 Subject: [PATCH 30/39] Remove local source type Additionally, ensure that the defaults are filled correctly for the `context` source type (which made the `local` type redundant). Signed-off-by: Peter Engelbert --- docs/testing.md | 1 - frontend/rpm/handle_sources.go | 4 +-- load.go | 47 ++++++++++++++++++++++++++------- source.go | 27 ++++++------------- spec.go | 10 +++---- test/fixtures/cmd-src-ref.yml | 2 +- test/fixtures/frontend.yml | 4 +-- test/fixtures/local-context.yml | 12 ++++----- test/fixtures/nested.yml | 2 +- 9 files changed, 59 insertions(+), 50 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 8459ee1f3..53f3ffcec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -231,7 +231,6 @@ tests: - dest: /target/mount/path spec: - ref: build:// build: inline: | FROM busybox diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index 5f73e563c..ef4d78fdb 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -99,9 +99,7 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err case src.Context != nil: s = src.Context.Name case src.Build != nil: - s = src.Build.Context - case src.Local != nil: - s = src.Local.Path + s = src.Build.ContextPath default: } diff --git a/load.go b/load.go index d7061ef1e..c86b849a1 100644 --- a/load.go +++ b/load.go @@ -6,6 +6,7 @@ import ( "github.com/goccy/go-yaml" "github.com/moby/buildkit/frontend/dockerfile/shell" + "github.com/moby/buildkit/frontend/dockerui" "github.com/pkg/errors" ) @@ -34,14 +35,21 @@ func (s *Source) processArgs(args map[string]string) error { case s.HTTPS != nil: sub = &s.HTTPS.URL case s.Context != nil: - sub = &s.Context.Name - if *sub == "" { - *sub = "context" + updated, err := lex.ProcessWordWithMap(s.Context.Name, args) + if err != nil { + return err + } + s.Context.Name = updated + + updated, err = lex.ProcessWordWithMap(s.Context.Path, args) + if err != nil { + return err } + s.Context.Path = updated + + return nil case s.Build != nil: - sub = &s.Build.Context - case s.Local != nil: - sub = &s.Local.Path + sub = &s.Build.ContextPath default: } @@ -58,6 +66,26 @@ func (s *Source) processArgs(args map[string]string) error { return nil } +func fillDefaults(s *Source) { + switch { + case s.DockerImage != nil: + case s.Git != nil: + case s.HTTPS != nil: + case s.Context != nil: + if s.Context.Name == "" { + s.Context.Name = dockerui.DefaultLocalNameContext + } + if s.Context.Path == "" { + s.Context.Path = "." + } + case s.Build != nil: + if s.Build.ContextPath == "" { + s.Build.ContextPath = "." + } + default: + } +} + func (s *Source) validate() error { count := 0 @@ -76,9 +104,6 @@ func (s *Source) validate() error { if s.Build != nil { count++ } - if s.Local != nil { - count++ - } var err error switch count { @@ -121,8 +146,10 @@ func LoadSpec(dt []byte, env map[string]string) (*Spec, error) { return nil, fmt.Errorf("error validating source ref %q: %w", name, err) } + fillDefaults(&src) + if err := src.processArgs(args); err != nil { - return nil, fmt.Errorf("error performing shell expansion on source ref %q: %w", name, err) + return nil, fmt.Errorf("error performing shell expansion on source %q: %w", name, err) } if src.DockerImage != nil { if err := src.DockerImage.Cmd.processBuildArgs(lex, args, name); err != nil { diff --git a/source.go b/source.go index fa6fe67f9..68bf010d5 100644 --- a/source.go +++ b/source.go @@ -194,24 +194,16 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter } includeExcludeHandled = true - if src.Path == "" && srcCtx.Name != "" { - src.Path = srcCtx.Name - } - return *st, nil - case src.Local != nil: - srcLocal := src.Local - - includeExcludeHandled = true - if src.Path == "" && srcLocal.Path != "" { - src.Path = srcLocal.Path + if src.Path == "" && srcCtx.Path != "" { + src.Path = srcCtx.Path } - return llb.Local(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)), nil + return *st, nil case src.Build != nil: build := src.Build var st llb.State - if build.Context == "" { + if build.ContextPath == "" { st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) } else { ctxState, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) @@ -221,8 +213,8 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter cst := *ctxState src2 := src - if src2.Path == "" && build.Context != "" { - src2.Path = build.Context + if src2.Path == "" && build.ContextPath != "" { + src2.Path = build.ContextPath } // This is necessary to have the specified context to be at the @@ -276,8 +268,7 @@ func SourceIsDir(src Source) (bool, error) { case src.DockerImage != nil, src.Git != nil, src.Build != nil, - src.Context != nil, - src.Local != nil: + src.Context != nil: return true, nil case src.HTTPS != nil: return false, nil @@ -294,13 +285,11 @@ func (s Source) Doc() (io.Reader, error) { switch { case s.Context != nil: fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") - case s.Local != nil: - fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") case s.Build != nil: build := s.Build fmt.Fprintln(b, "Generated from a docker build:") fmt.Fprintln(b, " Docker Build Target:", s.Build.Target) - fmt.Fprintln(b, " Docker Build Ref:", build.Context) + fmt.Fprintln(b, " Docker Build Ref:", build.ContextPath) if len(s.Build.Args) > 0 { sorted := SortMapKeys(s.Build.Args) diff --git a/spec.go b/spec.go index fe6a4cff1..eadb6e47d 100644 --- a/spec.go +++ b/spec.go @@ -195,14 +195,15 @@ type SourceHTTPS struct { } type SourceContext struct { - Name string `yaml:"name" json:"name"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Path string `yaml:"path,omitempty" json:"path,omitempty"` } // SourceBuild is used to generate source from a DockerFile build, either // inline or from a local file. type SourceBuild struct { // The local path to use as the context for the Dockerfile build - Context string `yaml:"context,omitempty" json:"context,omitempty"` + ContextPath string `yaml:"contextPath,omitempty" json:"contextPath,omitempty"` // DockerFile is the path to the build file in the build context // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. @@ -241,10 +242,6 @@ type Command struct { Steps []*BuildStep `yaml:"steps" json:"steps" jsonschema:"required"` } -type SourceLocal struct { - Path string `yaml:"path,omitempty" json:"path,omitempty"` -} - // Source defines a source to be used in the build. // A source can be a local directory, a git repositoryt, http(s) URL, etc. type Source struct { @@ -257,7 +254,6 @@ type Source struct { HTTPS *SourceHTTPS `yaml:"https,omitempty" json:"https,omitempty"` Context *SourceContext `yaml:"context,omitempty" json:"context,omitempty"` Build *SourceBuild `yaml:"build,omitempty" json:"build,omitempty"` - Local *SourceLocal `yaml:"local,omitempty" json:"local,omitempty"` // === End Source Variants === // Path is the path to the source after fetching it based on the identifier. diff --git a/test/fixtures/cmd-src-ref.yml b/test/fixtures/cmd-src-ref.yml index e70b1a2ac..d1a2c5314 100644 --- a/test/fixtures/cmd-src-ref.yml +++ b/test/fixtures/cmd-src-ref.yml @@ -22,7 +22,7 @@ sources: - dest: "/foo" spec: context: - name: "" + path: . steps: - env: FILE_TO_IMPORT: ${file_to_import} diff --git a/test/fixtures/frontend.yml b/test/fixtures/frontend.yml index 954626539..9dccb7681 100644 --- a/test/fixtures/frontend.yml +++ b/test/fixtures/frontend.yml @@ -11,7 +11,7 @@ packager: Microsoft license: Apache 2.0 sources: src: - local: + context: path: . gomodcache: path: /build/gomodcache @@ -22,7 +22,7 @@ sources: mounts: - dest: /build/src spec: - local: + context: path: . steps: - command: go mod download diff --git a/test/fixtures/local-context.yml b/test/fixtures/local-context.yml index 58f3b00e7..b2d06c8a1 100644 --- a/test/fixtures/local-context.yml +++ b/test/fixtures/local-context.yml @@ -1,4 +1,4 @@ -# syntax=ghcr.io/azure/dalec/frontend:latest +# syntax=local/dalec/frontend:latest name: dalec-test-fixutre-local-context description: A test fixture for checking local context sources @@ -12,25 +12,25 @@ license: Apache 2.0 sources: src: context: - name: "." + path: . src2: context: - name: "go.mod" + path: "go.mod" src3: context: - name: "." + path: . includes: - go.mod - go.sum src4: context: - name: "." + path: . excludes: - go.mod - go.sum src5: context: - name: "." + path: . includes: - go.* excludes: diff --git a/test/fixtures/nested.yml b/test/fixtures/nested.yml index a41030240..9701dd0ca 100644 --- a/test/fixtures/nested.yml +++ b/test/fixtures/nested.yml @@ -13,7 +13,7 @@ noarch: true sources: src: build: - context: test/fixtures/nested + contextPath: test/fixtures/nested inline: | # syntax=docker/dockerfile:1.5 FROM scratch From 4519995e4c5d8343c6f17844027b2aaf829d8049 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 12:52:23 -0500 Subject: [PATCH 31/39] Remove unused function We're no longer using the schemes Signed-off-by: Peter Engelbert --- source.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source.go b/source.go index 68bf010d5..81db55c46 100644 --- a/source.go +++ b/source.go @@ -249,14 +249,6 @@ func sharingMode(mode string) (llb.CacheMountSharingMode, error) { } } -func SplitSourceRef(ref string) (string, string, error) { - scheme, ref, ok := strings.Cut(ref, "://") - if !ok { - return "", "", fmt.Errorf("invalid source ref: %s", ref) - } - return scheme, ref, nil -} - func WithCreateDestPath() llb.CopyOption { return copyOptionFunc(func(i *llb.CopyInfo) { i.CreateDestPath = true From e40bf1065a052a8891292439267e6955dcf3994f Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 12:53:13 -0500 Subject: [PATCH 32/39] Run `go generate` Signed-off-by: Peter Engelbert --- docs/spec.schema.json | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/docs/spec.schema.json b/docs/spec.schema.json index 269e6834d..05d641ccf 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -424,9 +424,6 @@ "build": { "$ref": "#/$defs/SourceBuild" }, - "local": { - "$ref": "#/$defs/SourceLocal" - }, "path": { "type": "string", "description": "Path is the path to the source after fetching it based on the identifier." @@ -452,7 +449,7 @@ }, "SourceBuild": { "properties": { - "context": { + "contextPath": { "type": "string", "description": "The local path to use as the context for the Dockerfile build" }, @@ -487,13 +484,13 @@ "properties": { "name": { "type": "string" + }, + "path": { + "type": "string" } }, "additionalProperties": false, - "type": "object", - "required": [ - "name" - ] + "type": "object" }, "SourceDockerImage": { "properties": { @@ -543,15 +540,6 @@ ], "description": "No longer supports `.git` URLs as git repos." }, - "SourceLocal": { - "properties": { - "path": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SourceMount": { "properties": { "dest": { From 838074724496c79188986f8c232fe3052b4e73bd Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 13:30:07 -0500 Subject: [PATCH 33/39] Use strings for DockerFile/Inline on build sources Use them instead of pointers. Also, validate them up-front in the `validate` function, so that they do not need to be checked later. Signed-off-by: Peter Engelbert --- frontend/gateway.go | 13 ++++--------- load.go | 14 +++++++++----- source.go | 10 +++++----- spec.go | 5 ++--- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/frontend/gateway.go b/frontend/gateway.go index 717d08bce..f9d821b91 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -3,7 +3,6 @@ package frontend import ( "context" "encoding/json" - "fmt" "sync" "sync/atomic" @@ -29,10 +28,10 @@ func getDockerfile(ctx context.Context, client gwclient.Client, build *dalec.Sou dockerfilePath := dockerui.DefaultDockerfileName switch { - case build.Inline != nil: - return []byte(*build.Inline), nil - case build.DockerFile != nil: - dockerfilePath = *build.DockerFile + case build.Inline != "": + return []byte(build.Inline), nil + case build.DockerFile != "": + dockerfilePath = build.DockerFile } // First we need to read the dockerfile to determine what frontend to forward to @@ -65,10 +64,6 @@ func ForwarderFromClient(ctx context.Context, client gwclient.Client) dalec.Forw spec = &dalec.SourceBuild{} } - if spec.DockerFile != nil && spec.Inline != nil { - return llb.Scratch(), fmt.Errorf("cannot specify both file and inline for build spec") - } - def, err := st.Marshal(ctx) if err != nil { return llb.Scratch(), err diff --git a/load.go b/load.go index c86b849a1..57eef5749 100644 --- a/load.go +++ b/load.go @@ -1,6 +1,7 @@ package dalec import ( + goerrors "errors" "fmt" "path" @@ -88,6 +89,7 @@ func fillDefaults(s *Source) { func (s *Source) validate() error { count := 0 + var errs error if s.DockerImage != nil { count++ @@ -102,20 +104,22 @@ func (s *Source) validate() error { count++ } if s.Build != nil { + if s.Build.DockerFile != "" && s.Build.Inline != "" { + errs = goerrors.Join(errs, fmt.Errorf("build sources may use either `dockerfile` or `inline`, but not both")) + } count++ } - var err error switch count { case 0: - err = fmt.Errorf("no non-nil source variant") + errs = goerrors.Join(errs, fmt.Errorf("no non-nil source variant")) case 1: - err = nil + // success condition default: - err = fmt.Errorf("more than one source variant defined") + errs = goerrors.Join(errs, fmt.Errorf("more than one source variant defined")) } - return err + return errs } // LoadSpec loads a spec from the given data. diff --git a/source.go b/source.go index 81db55c46..fe4c5f989 100644 --- a/source.go +++ b/source.go @@ -292,20 +292,20 @@ func (s Source) Doc() (io.Reader, error) { } switch { - case s.Build.Inline != nil: + case s.Build.Inline != "": fmt.Fprintln(b, " Dockerfile:") - scanner := bufio.NewScanner(strings.NewReader(*s.Build.Inline)) + scanner := bufio.NewScanner(strings.NewReader(s.Build.Inline)) for scanner.Scan() { fmt.Fprintf(b, " %s\n", scanner.Text()) } if scanner.Err() != nil { return nil, scanner.Err() } - case s.Build.DockerFile != nil: + default: p := "Dockerfile" - if s.Build.DockerFile != nil { - p = *s.Build.DockerFile + if s.Build.DockerFile != "" { + p = s.Build.DockerFile } fmt.Fprintln(b, " Dockerfile path in context:", p) } diff --git a/spec.go b/spec.go index eadb6e47d..83484ac48 100644 --- a/spec.go +++ b/spec.go @@ -208,12 +208,11 @@ type SourceBuild struct { // DockerFile is the path to the build file in the build context // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. // This is exclusive with [Inline] - DockerFile *string `yaml:"dockerfile,omitempty" json:"dockerfile,omitempty"` + DockerFile string `yaml:"dockerfile,omitempty" json:"dockerfile,omitempty"` // Inline is an inline build spec to use. // This can be used to specify a dockerfile instead of using one in the build context // This is exclusive with [File] - Inline *string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` - // === End SourceBuild Action Variants === + Inline string `yaml:"inline,omitempty" json:"inline,omitempty" jsonschema:"example=FROM busybox\nRUN echo hello world"` // Target specifies the build target to use. // If unset, the default target is determined by the frontend implementation From a3ddd8a898d4497d00d54fb686be3f5f5528e014 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 13:34:14 -0500 Subject: [PATCH 34/39] Add error to default case Signed-off-by: Peter Engelbert --- frontend/rpm/handle_sources.go | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index ef4d78fdb..d717ad601 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -101,6 +101,7 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err case src.Build != nil: s = src.Build.ContextPath default: + return nil, fmt.Errorf("no non-nil source provided") } pg := llb.ProgressGroup(pgID, "Add spec source: "+k+" "+s, false) From 140a9a9a331f94121bf669bbca6a2f493fa44f55 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 16:10:48 -0500 Subject: [PATCH 35/39] Allow `Build` Source type to use any Source as the context Signed-off-by: Peter Engelbert --- docs/spec.schema.json | 6 ++--- frontend/rpm/handle_sources.go | 2 +- load.go | 33 ++++++++++++++++++++++---- source.go | 43 +++++++++++++--------------------- spec.go | 4 ++-- test/fixtures/nested.yml | 4 +++- 6 files changed, 53 insertions(+), 39 deletions(-) diff --git a/docs/spec.schema.json b/docs/spec.schema.json index 05d641ccf..a5a02bd29 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -449,9 +449,9 @@ }, "SourceBuild": { "properties": { - "contextPath": { - "type": "string", - "description": "The local path to use as the context for the Dockerfile build" + "source": { + "$ref": "#/$defs/Source", + "description": "A source specification to use as the context for the Dockerfile build" }, "dockerfile": { "type": "string", diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index d717ad601..f17eb1c95 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -99,7 +99,7 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts) ([]llb.State, err case src.Context != nil: s = src.Context.Name case src.Build != nil: - s = src.Build.ContextPath + s = fmt.Sprintf("%v", src.Build.Source) default: return nil, fmt.Errorf("no non-nil source provided") } diff --git a/load.go b/load.go index 57eef5749..42b7533eb 100644 --- a/load.go +++ b/load.go @@ -30,6 +30,9 @@ func (s *Source) processArgs(args map[string]string) error { var sub *string switch { case s.DockerImage != nil: + for _, mnt := range s.DockerImage.Cmd.Mounts { + mnt.Spec.processArgs(args) + } sub = &s.DockerImage.Ref case s.Git != nil: sub = &s.Git.URL @@ -48,9 +51,23 @@ func (s *Source) processArgs(args map[string]string) error { } s.Context.Path = updated - return nil + sub = nil case s.Build != nil: - sub = &s.Build.ContextPath + s.Build.Source.processArgs(args) + + updated, err := lex.ProcessWordWithMap(s.Build.DockerFile, args) + if err != nil { + return err + } + s.Build.DockerFile = updated + + updated, err = lex.ProcessWordWithMap(s.Build.Target, args) + if err != nil { + return err + } + s.Build.Target = updated + + sub = nil default: } @@ -70,6 +87,9 @@ func (s *Source) processArgs(args map[string]string) error { func fillDefaults(s *Source) { switch { case s.DockerImage != nil: + for _, mnt := range s.DockerImage.Cmd.Mounts { + fillDefaults(&mnt.Spec) + } case s.Git != nil: case s.HTTPS != nil: case s.Context != nil: @@ -80,9 +100,7 @@ func fillDefaults(s *Source) { s.Context.Path = "." } case s.Build != nil: - if s.Build.ContextPath == "" { - s.Build.ContextPath = "." - } + fillDefaults(&s.Build.Source) default: } } @@ -107,6 +125,11 @@ func (s *Source) validate() error { if s.Build.DockerFile != "" && s.Build.Inline != "" { errs = goerrors.Join(errs, fmt.Errorf("build sources may use either `dockerfile` or `inline`, but not both")) } + + if s.Build.Source.Build != nil { + errs = goerrors.Join(errs, fmt.Errorf("build sources cannot be recursive")) + } + count++ } diff --git a/source.go b/source.go index fe4c5f989..71cf3b00d 100644 --- a/source.go +++ b/source.go @@ -201,32 +201,11 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter return *st, nil case src.Build != nil: build := src.Build - var st llb.State - - if build.ContextPath == "" { - st = llb.Local(dockerui.DefaultLocalNameContext, withConstraints(opts)) - } else { - ctxState, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) - if err != nil { - return llb.Scratch(), err - } - cst := *ctxState - - src2 := src - if src2.Path == "" && build.ContextPath != "" { - src2.Path = build.ContextPath - } + subSource := build.Source - // This is necessary to have the specified context to be at the - // root of the state's fs. - st, _ = handleFilter(&filterOpts{ - state: cst, - source: src2, - opts: opts, - forMount: forMount, - includeExcludeHandled: true, - pathHandled: false, - }) + st, err := source2LLBGetter(s, subSource, name, forMount)(sOpt, opts...) + if err != nil { + return llb.Scratch(), err } return sOpt.Forward(st, build) @@ -278,10 +257,20 @@ func (s Source) Doc() (io.Reader, error) { case s.Context != nil: fmt.Fprintln(b, "Generated from a local docker build context and is unreproducible.") case s.Build != nil: - build := s.Build fmt.Fprintln(b, "Generated from a docker build:") fmt.Fprintln(b, " Docker Build Target:", s.Build.Target) - fmt.Fprintln(b, " Docker Build Ref:", build.ContextPath) + sub, err := s.Build.Source.Doc() + if err != nil { + return nil, err + } + + scanner := bufio.NewScanner(sub) + for scanner.Scan() { + fmt.Fprintf(b, " %s\n", scanner.Text()) + } + if scanner.Err() != nil { + return nil, scanner.Err() + } if len(s.Build.Args) > 0 { sorted := SortMapKeys(s.Build.Args) diff --git a/spec.go b/spec.go index 83484ac48..698b8c40d 100644 --- a/spec.go +++ b/spec.go @@ -202,8 +202,8 @@ type SourceContext struct { // SourceBuild is used to generate source from a DockerFile build, either // inline or from a local file. type SourceBuild struct { - // The local path to use as the context for the Dockerfile build - ContextPath string `yaml:"contextPath,omitempty" json:"contextPath,omitempty"` + // A source specification to use as the context for the Dockerfile build + Source Source `yaml:"source,omitempty" json:"source,omitempty"` // DockerFile is the path to the build file in the build context // If not set the default is assumed by buildkit to be `Dockerfile` at the root of the context. diff --git a/test/fixtures/nested.yml b/test/fixtures/nested.yml index 9701dd0ca..1bf24e064 100644 --- a/test/fixtures/nested.yml +++ b/test/fixtures/nested.yml @@ -13,7 +13,9 @@ noarch: true sources: src: build: - contextPath: test/fixtures/nested + source: + context: + path: test/fixtures/nested inline: | # syntax=docker/dockerfile:1.5 FROM scratch From 1a864c1c41a6ac6f1837d2e1a77cf27b97e4ba95 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 10 Jan 2024 16:19:16 -0500 Subject: [PATCH 36/39] Address linter errors Also, validate sources recursively. Signed-off-by: Peter Engelbert --- load.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/load.go b/load.go index 42b7533eb..a48b4be37 100644 --- a/load.go +++ b/load.go @@ -31,7 +31,9 @@ func (s *Source) processArgs(args map[string]string) error { switch { case s.DockerImage != nil: for _, mnt := range s.DockerImage.Cmd.Mounts { - mnt.Spec.processArgs(args) + if err := mnt.Spec.processArgs(args); err != nil { + return err + } } sub = &s.DockerImage.Ref case s.Git != nil: @@ -53,7 +55,9 @@ func (s *Source) processArgs(args map[string]string) error { sub = nil case s.Build != nil: - s.Build.Source.processArgs(args) + if err := s.Build.Source.processArgs(args); err != nil { + return err + } updated, err := lex.ProcessWordWithMap(s.Build.DockerFile, args) if err != nil { @@ -110,6 +114,12 @@ func (s *Source) validate() error { var errs error if s.DockerImage != nil { + for _, mnt := range s.DockerImage.Cmd.Mounts { + if err := mnt.Spec.validate(); err != nil { + errs = goerrors.Join(errs, err) + } + } + count++ } if s.Git != nil { @@ -122,12 +132,16 @@ func (s *Source) validate() error { count++ } if s.Build != nil { + if s.Build.Source.Build != nil { + errs = goerrors.Join(errs, fmt.Errorf("build sources cannot be recursive")) + } + if s.Build.DockerFile != "" && s.Build.Inline != "" { errs = goerrors.Join(errs, fmt.Errorf("build sources may use either `dockerfile` or `inline`, but not both")) } - if s.Build.Source.Build != nil { - errs = goerrors.Join(errs, fmt.Errorf("build sources cannot be recursive")) + if err := s.Build.Source.validate(); err != nil { + errs = goerrors.Join(errs, err) } count++ From d76f447d94f84c36c982c1c06bfc45b2c93dba5c Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 12 Jan 2024 11:40:04 -0500 Subject: [PATCH 37/39] Remove `Path` field from `SourceContext` Also: - update `processArgs` to do substitutions inline. - update `fillDefaults` to set the default Source.Path to `"."` when the Source type is Context. Signed-off-by: Peter Engelbert --- docs/spec.schema.json | 9 +++--- load.go | 70 ++++++++++++++++++++----------------------- source.go | 6 ---- spec.go | 5 +++- 4 files changed, 41 insertions(+), 49 deletions(-) diff --git a/docs/spec.schema.json b/docs/spec.schema.json index a5a02bd29..feaf930f6 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -483,14 +483,13 @@ "SourceContext": { "properties": { "name": { - "type": "string" - }, - "path": { - "type": "string" + "type": "string", + "description": "Name is the name of the build context. By default, it is the magic name\n`context`, recognized by Docker as the default context." } }, "additionalProperties": false, - "type": "object" + "type": "object", + "description": "SourceContext is used to generate a source from a build context." }, "SourceDockerImage": { "properties": { diff --git a/load.go b/load.go index a48b4be37..96e4c7666 100644 --- a/load.go +++ b/load.go @@ -27,7 +27,15 @@ const DefaultPatchStrip int = 1 func (s *Source) processArgs(args map[string]string) error { lex := shell.NewLex('\\') - var sub *string + sub := func(s *string) error { + updated, err := lex.ProcessWordWithMap(*s, args) + if err != nil { + return err + } + *s = updated + return nil + } + switch { case s.DockerImage != nil: for _, mnt := range s.DockerImage.Cmd.Mounts { @@ -35,56 +43,44 @@ func (s *Source) processArgs(args map[string]string) error { return err } } - sub = &s.DockerImage.Ref + if err := sub(&s.DockerImage.Ref); err != nil { + return err + } case s.Git != nil: - sub = &s.Git.URL + fields := []*string{ + &s.Git.URL, + &s.Git.Commit, + } + for _, f := range fields { + if err := sub(f); err != nil { + return err + } + } case s.HTTPS != nil: - sub = &s.HTTPS.URL - case s.Context != nil: - updated, err := lex.ProcessWordWithMap(s.Context.Name, args) - if err != nil { + if err := sub(&s.HTTPS.URL); err != nil { return err } - s.Context.Name = updated - - updated, err = lex.ProcessWordWithMap(s.Context.Path, args) - if err != nil { + case s.Context != nil: + if err := sub(&s.Context.Name); err != nil { return err } - s.Context.Path = updated - - sub = nil case s.Build != nil: if err := s.Build.Source.processArgs(args); err != nil { return err } - updated, err := lex.ProcessWordWithMap(s.Build.DockerFile, args) - if err != nil { - return err + fields := []*string{ + &s.Build.DockerFile, + &s.Build.Target, } - s.Build.DockerFile = updated - - updated, err = lex.ProcessWordWithMap(s.Build.Target, args) - if err != nil { - return err + for _, f := range fields { + if err := sub(f); err != nil { + return err + } } - s.Build.Target = updated - - sub = nil default: } - if sub == nil { - return nil - } - - updated, err := lex.ProcessWordWithMap(*sub, args) - if err != nil { - return err - } - - *sub = updated return nil } @@ -100,8 +96,8 @@ func fillDefaults(s *Source) { if s.Context.Name == "" { s.Context.Name = dockerui.DefaultLocalNameContext } - if s.Context.Path == "" { - s.Context.Path = "." + if s.Path == "" { + s.Path = "." } case s.Build != nil: fillDefaults(&s.Build.Source) diff --git a/source.go b/source.go index 71cf3b00d..fec59305c 100644 --- a/source.go +++ b/source.go @@ -186,18 +186,12 @@ func source2LLBGetter(s *Spec, src Source, name string, forMount bool) LLBGetter opts = append(opts, llb.Filename(name)) return llb.HTTP(https.URL, opts...), nil case src.Context != nil: - srcCtx := src.Context - st, err := sOpt.GetContext(dockerui.DefaultLocalNameContext, localIncludeExcludeMerge(&src)) if err != nil { return llb.Scratch(), err } includeExcludeHandled = true - if src.Path == "" && srcCtx.Path != "" { - src.Path = srcCtx.Path - } - return *st, nil case src.Build != nil: build := src.Build diff --git a/spec.go b/spec.go index 698b8c40d..349690ea1 100644 --- a/spec.go +++ b/spec.go @@ -194,9 +194,12 @@ type SourceHTTPS struct { URL string `yaml:"url" json:"url"` } +// SourceContext is used to generate a source from a build context. The path to +// the build context is provided to the `Path` field of the owning `Source`. type SourceContext struct { + // Name is the name of the build context. By default, it is the magic name + // `context`, recognized by Docker as the default context. Name string `yaml:"name,omitempty" json:"name,omitempty"` - Path string `yaml:"path,omitempty" json:"path,omitempty"` } // SourceBuild is used to generate source from a DockerFile build, either From c2a82303c3774cb5e78d1b1ac589a200d6487f5b Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 12 Jan 2024 11:51:31 -0500 Subject: [PATCH 38/39] Make yaml specs conform to Git source type layout Signed-off-by: Peter Engelbert --- docs/examples/go-md2man-1.yml | 3 ++- docs/examples/go-md2man-2.yml | 6 ++++-- docs/intro.md | 9 ++++++--- test/fixtures/kubernetes-patch.yml | 3 ++- test/fixtures/moby-runc.yml | 3 ++- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/examples/go-md2man-1.yml b/docs/examples/go-md2man-1.yml index 4d965e96d..db3b8a2b5 100644 --- a/docs/examples/go-md2man-1.yml +++ b/docs/examples/go-md2man-1.yml @@ -10,7 +10,8 @@ website: https://github.com/cpuguy83/go-md2man sources: src: git: - url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + url: https://github.com/cpuguy83/go-md2man.git + commit: "v2.0.3" dependencies: build: diff --git a/docs/examples/go-md2man-2.yml b/docs/examples/go-md2man-2.yml index 8afffcfc3..a20a63564 100644 --- a/docs/examples/go-md2man-2.yml +++ b/docs/examples/go-md2man-2.yml @@ -10,7 +10,8 @@ website: https://github.com/cpuguy83/go-md2man sources: src: git: - url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + url: https://github.com/cpuguy83/go-md2man.git + commit: "v2.0.3" gomods: # This is required when the build environment does not allow network access. This downloads all the go modules. path: /build/gomodcache # This is the path we will be extracing after running the command below. image: @@ -22,7 +23,8 @@ sources: - dest: /build/src spec: git: - url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + url: https://github.com/cpuguy83/go-md2man.git + commit: "v2.0.3" steps: - command: go mod download env: diff --git a/docs/intro.md b/docs/intro.md index 06ab6f03b..807edf3fd 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -139,7 +139,8 @@ website: https://github.com/cpuguy83/go-md2man sources: src: git: - url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + url: https://github.com/cpuguy83/go-md2man.git + commit: "v2.0.3" dependencies: build: @@ -203,7 +204,8 @@ website: https://github.com/cpuguy83/go-md2man sources: src: git: - url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + url: https://github.com/cpuguy83/go-md2man.git + commit: "v2.0.3" gomods: # This is required when the build environment does not allow network access. This downloads all the go modules. path: /build/gomodcache # This is the path we will be extracing after running the command below. image: @@ -215,7 +217,8 @@ sources: - dest: /build/src spec: git: - url: https://github.com/cpuguy83/go-md2man.git#v2.0.3 + url: https://github.com/cpuguy83/go-md2man.git + commit: "v2.0.3" steps: - command: go mod download env: diff --git a/test/fixtures/kubernetes-patch.yml b/test/fixtures/kubernetes-patch.yml index 3681c8ee3..e13a65aea 100644 --- a/test/fixtures/kubernetes-patch.yml +++ b/test/fixtures/kubernetes-patch.yml @@ -15,7 +15,8 @@ dependencies: sources: src: git: - url: https://github.com/kubernetes/kubernetes.git#v1.28.0 + url: https://github.com/kubernetes/kubernetes.git + commit: "v1.28.0" patch_120129: https: url: https://patch-diff.githubusercontent.com/raw/kubernetes/kubernetes/pull/120129.patch diff --git a/test/fixtures/moby-runc.yml b/test/fixtures/moby-runc.yml index f6a23ce6f..d6600de30 100644 --- a/test/fixtures/moby-runc.yml +++ b/test/fixtures/moby-runc.yml @@ -62,7 +62,8 @@ provides: sources: src: git: - url: https://github.com/opencontainers/runc.git#${RUNC_COMMIT} + url: https://github.com/opencontainers/runc.git + commit: "${RUNC_COMMIT}" build: env: From b3a2642035da3e5ba80031ae256f3d2dc451641d Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Fri, 12 Jan 2024 11:52:56 -0500 Subject: [PATCH 39/39] Remove unused default cases from switch statements Signed-off-by: Peter Engelbert --- load.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/load.go b/load.go index 96e4c7666..c0c21b4a7 100644 --- a/load.go +++ b/load.go @@ -78,7 +78,6 @@ func (s *Source) processArgs(args map[string]string) error { return err } } - default: } return nil @@ -101,7 +100,6 @@ func fillDefaults(s *Source) { } case s.Build != nil: fillDefaults(&s.Build.Source) - default: } }