-
Notifications
You must be signed in to change notification settings - Fork 70
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Before only raw buildkitd unix socket or docker socket was supported. This adds support for: - Docker with custom socket path - Buildx instances (defaults to the currently selected buildx builder instance) - Connect over a docker container which works similarly to buildx but does not require a buildx config. For the buildx support, currently only buildx instances over docker-containers are supported. Support other buildx drivers is possible (e.g. the kubernetes driver), but just needs some working through what's there. Signed-off-by: Brian Goff <[email protected]>
- Loading branch information
Showing
5 changed files
with
271 additions
and
65 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,265 @@ | ||
| package buildkit | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "net" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "sync/atomic" | ||
|
|
||
| "github.com/cpuguy83/dockercfg" | ||
| "github.com/cpuguy83/go-docker" | ||
| "github.com/cpuguy83/go-docker/buildkitopt" | ||
| "github.com/cpuguy83/go-docker/container" | ||
| "github.com/cpuguy83/go-docker/transport" | ||
| "github.com/moby/buildkit/client" | ||
| gateway "github.com/moby/buildkit/frontend/gateway/client" | ||
| "github.com/moby/buildkit/solver/pb" | ||
| "github.com/moby/buildkit/util/apicaps" | ||
| log "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| const ( | ||
| DefaultAddr = "unix:///run/buildkit/buildkitd.sock" | ||
| ) | ||
|
|
||
| var ( | ||
| errMissingCap = fmt.Errorf("missing required buildkit functionality") | ||
| // requiredCaps are buildkit llb ops required to function. | ||
| requiredCaps = []apicaps.CapID{pb.CapMergeOp, pb.CapDiffOp} | ||
| ) | ||
|
|
||
| type driverFunc func(context.Context) (*client.Client, error) | ||
|
|
||
| func validateClient(ctx context.Context, c *client.Client) error { | ||
| _, err := c.Build(ctx, client.SolveOpt{}, "", func(ctx context.Context, client gateway.Client) (*gateway.Result, error) { | ||
| capset := client.BuildOpts().LLBCaps | ||
| var err error | ||
| for _, cap := range requiredCaps { | ||
| err = errors.Join(err, capset.Supports(cap)) | ||
| } | ||
| if err != nil { | ||
| return nil, errors.Join(err, errMissingCap) | ||
| } | ||
| return &gateway.Result{}, nil | ||
| }, nil) | ||
| return err | ||
| } | ||
|
|
||
| func withValidateClient(f driverFunc) driverFunc { | ||
| if f == nil { | ||
| return nil | ||
| } | ||
|
|
||
| return func(ctx context.Context) (*client.Client, error) { | ||
| client, err := f(ctx) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if err := validateClient(ctx, client); err != nil { | ||
| client.Close() | ||
| return nil, err | ||
| } | ||
| return client, nil | ||
| } | ||
| } | ||
|
|
||
| func getDriver(spec string) (ret driverFunc, _ error) { | ||
| defer func() { | ||
| ret = withValidateClient(ret) | ||
| }() | ||
|
|
||
| name, addr, ok := strings.Cut(spec, "://") | ||
| if !ok { | ||
| if spec != "" { | ||
| return nil, errors.New("invalid driver spec") | ||
| } | ||
| return autoDriver, nil | ||
| } | ||
|
|
||
| switch name { | ||
| case "docker": | ||
| return dockerDriver(addr), nil | ||
| case "buildx": | ||
| return buildxDriver(addr), nil | ||
| case "unix": | ||
| return buildkitdDriver(addr), nil | ||
| case "docker-container": | ||
| if addr == "" { | ||
| return nil, errors.New("docker-container driver requires a container name or id") | ||
| } | ||
| return dockerContainerDriver("", addr), nil | ||
| } | ||
|
|
||
| return nil, errors.New("invalid driver") | ||
| } | ||
|
|
||
| func buildxDriver(addr string) driverFunc { | ||
| return func(ctx context.Context) (*client.Client, error) { | ||
| configPath, err := dockercfg.ConfigPath() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| base := filepath.Join(filepath.Dir(configPath), "buildx") | ||
| if addr == "" { | ||
| dt, err := os.ReadFile(filepath.Join(base, "current")) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| type ref struct { | ||
| Name string `json:"name"` | ||
| } | ||
| var r ref | ||
| if err := json.Unmarshal(dt, &r); err != nil { | ||
| return nil, fmt.Errorf("could not unmarshal buildx config: %w", err) | ||
| } | ||
| addr = r.Name | ||
| } | ||
|
|
||
| dt, err := os.ReadFile(filepath.Join(base, "instances", addr)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| type buildxConfig struct { | ||
| Driver string | ||
| Nodes []struct { | ||
| Name string | ||
| Endpoint string | ||
| } | ||
| } | ||
| var cfg buildxConfig | ||
| if err := json.Unmarshal(dt, &cfg); err != nil { | ||
| return nil, fmt.Errorf("could not unmarshal buildx instance config: %w", err) | ||
| } | ||
|
|
||
| if cfg.Driver != "docker-container" { | ||
| return nil, fmt.Errorf("unsupported buildx driver: %s", cfg.Driver) | ||
| } | ||
|
|
||
| if len(cfg.Nodes) == 0 { | ||
| return nil, errors.New("no nodes configured for buildx instance") | ||
| } | ||
|
|
||
| log.WithFields(log.Fields{ | ||
| "driver": cfg.Driver, | ||
| "endpoint": cfg.Nodes[0].Endpoint, | ||
| "name": cfg.Nodes[0].Name, | ||
| }).Debug("Connect to buildx instance") | ||
| return dockerContainerDriver(cfg.Nodes[0].Endpoint, "buildx_buildkit_"+cfg.Nodes[0].Name)(ctx) | ||
| } | ||
| } | ||
|
|
||
| func autoDriver(ctx context.Context) (*client.Client, error) { | ||
| var retErr error | ||
|
|
||
| log.Debug("Tryin docker driver") | ||
| client, err := dockerDriver("")(ctx) | ||
| if err == nil { | ||
| err = validateClient(ctx, client) | ||
| if err == nil { | ||
| return client, nil | ||
| } | ||
| client.Close() | ||
| } | ||
| retErr = errors.Join(retErr, fmt.Errorf("could not use docker driver: %w", err)) | ||
|
|
||
| log.Debug("Trying buildx driver") | ||
| client, err = buildxDriver("")(ctx) | ||
| if err == nil { | ||
| err = validateClient(ctx, client) | ||
| if err == nil { | ||
| return client, nil | ||
| } | ||
| client.Close() | ||
| } | ||
| retErr = errors.Join(retErr, fmt.Errorf("could not use buildx driver: %w", err)) | ||
|
|
||
| log.Debug("Trying buildkitd driver") | ||
| client, err = buildkitdDriver("")(ctx) | ||
| if err == nil { | ||
| if err := validateClient(ctx, client); err == nil { | ||
| return client, nil | ||
| } | ||
| client.Close() | ||
| } | ||
| return nil, errors.Join(retErr, fmt.Errorf("could not use buildkitd driver: %w", err)) | ||
| } | ||
|
|
||
| func buildkitdDriver(addr string) driverFunc { | ||
| return func(ctx context.Context) (*client.Client, error) { | ||
| if addr == "" { | ||
| addr = DefaultAddr | ||
| } | ||
| return client.New(ctx, "unix://"+addr, client.WithFailFast()) | ||
| } | ||
| } | ||
|
|
||
| func getDockerTransport(addr string) (transport.Doer, error) { | ||
| if addr == "" { | ||
| addr = os.Getenv("DOCKER_HOST") | ||
| } | ||
|
|
||
| if addr == "" { | ||
| return transport.DefaultTransport() | ||
| } | ||
| return transport.FromConnectionString(addr) | ||
| } | ||
|
|
||
| func dockerDriver(addr string) driverFunc { | ||
| return func(ctx context.Context) (*client.Client, error) { | ||
| defaultOpts := []client.ClientOpt{client.WithFailFast()} | ||
| bkOpts := defaultOpts | ||
|
|
||
| tr, err := getDockerTransport(addr) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| bkOpts = append(bkOpts, buildkitopt.FromDocker(tr)...) | ||
| c, err := client.New(ctx, addr, bkOpts...) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return c, nil | ||
| } | ||
| } | ||
|
|
||
| func dockerContainerDriver(host, addr string) driverFunc { | ||
| return func(ctx context.Context) (*client.Client, error) { | ||
| tr, err := getDockerTransport(host) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| cli := docker.NewClient(docker.WithTransport(tr)) | ||
| c := cli.ContainerService().NewContainer(ctx, addr) | ||
|
|
||
| conn1, conn2 := net.Pipe() | ||
| ep, err := c.Exec(ctx, container.WithExecCmd("buildctl", "dial-stdio"), func(cfg *container.ExecConfig) { | ||
| cfg.Stdin = conn1 | ||
| cfg.Stdout = conn1 | ||
| cfg.Stderr = conn1 | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if err := ep.Start(ctx); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| var opts []client.ClientOpt | ||
| var counter int64 | ||
| opts = append(opts, client.WithContextDialer(func(context.Context, string) (net.Conn, error) { | ||
| if atomic.AddInt64(&counter, 1) > 1 { | ||
| return nil, net.ErrClosed | ||
| } | ||
| return conn2, nil | ||
| })) | ||
| return client.New(ctx, "", opts...) | ||
| } | ||
| } |