appdev: correct source_dir and dockerfile_path interaction (#1257)

* add dockerfile/buildpacks indicator to component list

* dockerfile builds: correct dockerfile_path and source_dir interaction

* support dockerfile_path outside of source_dir

* update tests

* fix windows path handling

commands/apps_charm.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/digitalocean/doctl/commands/charm"
 	"github.com/digitalocean/doctl/commands/charm/template"
+	"github.com/digitalocean/doctl/internal/apps/builder"
 	"github.com/digitalocean/godo"
 )
 
@@ -22,6 +23,12 @@ func (i componentListItem) Description() string {
 	}
 
 	if buildable, ok := i.spec.(godo.AppBuildableComponentSpec); ok {
+		if builder.IsDockerBuild(buildable) {
+			desc[0] += " [dockerfile]"
+		} else if builder.IsCNBBuild(buildable) {
+			desc[0] += " [buildpacks]"
+		}
+
 		if sourceDir := buildable.GetSourceDir(); sourceDir != "" {
 			desc = append(desc, template.String(`located in ./{{highlight .}}`, sourceDir))
 		}

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/mattn/go-isatty v0.0.14
 	github.com/mitchellh/copystructure v1.0.0
 	github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007
-	github.com/opencontainers/image-spec v1.0.2
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
 	github.com/opencontainers/runc v1.1.3 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/sclevine/spec v1.3.0
@@ -47,7 +47,6 @@ require (
 require (
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/apache/openwhisk-client-go v0.0.0-20211007130743-38709899040b
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
 	github.com/charmbracelet/bubbles v0.13.1-0.20220731172002-8f6516082803
 	github.com/charmbracelet/bubbletea v0.22.0
@@ -57,17 +56,18 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/muesli/reflow v0.3.0
 	github.com/muesli/termenv v0.12.0
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
 	github.com/containerd/cgroups v1.0.3 // indirect
 	github.com/containerd/console v1.0.3 // indirect
-	github.com/containerd/containerd v1.5.13 // indirect
+	github.com/containerd/containerd v1.6.3-0.20220401172941-5ff8fce1fcc6 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
@@ -91,9 +91,11 @@ require (
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mitchellh/reflectwalk v1.0.0 // indirect
+	github.com/moby/buildkit v0.10.4 // indirect
 	github.com/moby/sys/mount v0.3.3 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
-	github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect
+	github.com/moby/sys/symlink v0.2.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect

go.sum

@@ -38,8 +38,9 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
@@ -186,8 +187,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq
 github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
 github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
 github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
-github.com/containerd/containerd v1.5.13 h1:XqvKw9i4P7/mFrC3TSM7yV5cwFZ9avXe6M3YANKnzEE=
-github.com/containerd/containerd v1.5.13/go.mod h1:3AlCrzKROjIuP3JALsY14n8YtntaUDBu7vek+rPN5Vc=
+github.com/containerd/containerd v1.6.3-0.20220401172941-5ff8fce1fcc6 h1:nig7zto6cp3Wt1lPMK8EmyP6f/ZNmn/tL6ASQ7stews=
+github.com/containerd/containerd v1.6.3-0.20220401172941-5ff8fce1fcc6/go.mod h1:WSt2SnDLAGWlu+Vl+EWay37seZLKqgRt6XLjIMy8SYM=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -590,6 +591,8 @@ github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/buildkit v0.10.4 h1:FvC+buO8isGpUFZ1abdSLdGHZVqg9sqI4BbFL8tlzP4=
+github.com/moby/buildkit v0.10.4/go.mod h1:Yajz9vt1Zw5q9Pp4pdb3TCSUXJBIroIQGQ3TTs/sLug=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
@@ -600,8 +603,11 @@ github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx
 github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI=
+github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc=
+github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
+github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -671,8 +677,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
-github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
+github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=

internal/apps/builder/builder.go

@@ -322,3 +322,13 @@ func IsCNBBuild(spec godo.AppBuildableComponentSpec) bool {
 
 	return true
 }
+
+// IsDockerBuild indicates whether the component will be built using the Docker builder.
+func IsDockerBuild(spec godo.AppBuildableComponentSpec) bool {
+	dockerBuildable, ok := spec.(godo.AppDockerBuildableComponentSpec)
+	if !ok {
+		return false
+	}
+
+	return dockerBuildable.GetDockerfilePath() != ""
+}

internal/apps/builder/docker.go

@@ -16,8 +16,10 @@ import (
 
 	"github.com/digitalocean/doctl/commands/charm/template"
 	"github.com/digitalocean/godo"
+	"github.com/docker/cli/cli/command/image/build"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/idtools"
 )
 
 // DockerComponentBuilder builds components using a Dockerfile.
@@ -58,23 +60,20 @@ func (b *DockerComponentBuilder) Build(ctx context.Context) (ComponentBuilderRes
 		return ComponentBuilderResult{}, fmt.Errorf("configuring environment variables: %w", err)
 	}
 
-	buildContext := b.contextDir
-	if sd := filepath.Clean(b.component.GetSourceDir()); sd != "." && sd != "/" {
-		buildContext = filepath.Join(buildContext, sd)
-	}
-	// TODO Dockerfile must be relative to the source dir.
-	// Make it relative and if it's outside the source dir add it to the archive.
-	// ref: https://github.com/docker/cli/blob/9400e3dbe8ebd0bede3ab7023f744a8d7f4397d2/cli/command/image/build.go#L280-L286
-	template.Render(lw, `{{success checkmark}} building image using dockerfile {{highlight .}}{{nl 2}}`, b.dockerComponent.GetDockerfilePath())
+	template.Render(lw,
+		`{{success checkmark}} building image using dockerfile {{highlight .}}{{nl 2}}`,
+		b.dockerComponent.GetDockerfilePath(),
+	)
 	start := time.Now()
-	tar, err := archive.TarWithOptions(buildContext, &archive.TarOptions{})
+
+	imageBuildContext, imageBuildDockerfile, err := b.getImageBuildContext(ctx)
 	if err != nil {
 		return ComponentBuilderResult{}, fmt.Errorf("preparing build context: %w", err)
 	}
 
 	res := ComponentBuilderResult{}
-	dockerRes, err := b.cli.ImageBuild(ctx, tar, dockertypes.ImageBuildOptions{
-		Dockerfile: b.dockerComponent.GetDockerfilePath(),
+	dockerRes, err := b.cli.ImageBuild(ctx, imageBuildContext, dockertypes.ImageBuildOptions{
+		Dockerfile: imageBuildDockerfile,
 		Tags: []string{
 			b.AppImageOutputName(),
 		},
@@ -106,6 +105,65 @@ func (b *DockerComponentBuilder) Build(ctx context.Context) (ComponentBuilderRes
 	return res, nil
 }
 
+func (b *DockerComponentBuilder) getImageBuildContext(ctx context.Context) (io.Reader, string, error) {
+	// this assembles the build context in a way that fits cli.ImageBuild's expectations around
+	// dockerfiles and .dockerignore.
+	// much of this logic is copied from the `docker` cli implementation:
+	//   https://github.com/docker/cli/blob/9400e3dbe8ebd0bede3ab7023f744a8d7f4397d2/cli/command/image/build.go#L180
+	//   specifically the "build context is a local directory" flow.
+
+	absSourceDir, err := filepath.Abs(filepath.Join(b.contextDir, b.dockerComponent.GetSourceDir()))
+	if err != nil {
+		return nil, "", fmt.Errorf("parsing source_dir: %w", err)
+	}
+	absDockerfile, err := filepath.Abs(filepath.Join(b.contextDir, b.dockerComponent.GetDockerfilePath()))
+	if err != nil {
+		return nil, "", fmt.Errorf("parsing dockerfile_path: %w", err)
+	}
+	relDockerfile, err := filepath.Rel(absSourceDir, absDockerfile)
+	if err != nil {
+		return nil, "", err
+	}
+
+	excludes, err := build.ReadDockerignore(absSourceDir)
+	if err != nil {
+		return nil, "", fmt.Errorf("reading .dockerignore: %w", err)
+	}
+
+	if err := build.ValidateContextDirectory(absSourceDir, excludes); err != nil {
+		return nil, "", err
+	}
+
+	// canonicalize dockerfile name to a platform-independent one
+	relDockerfile = archive.CanonicalTarNameForPath(relDockerfile)
+	excludes = build.TrimBuildFilesFromExcludes(excludes, relDockerfile, false)
+	tar, err := archive.TarWithOptions(absSourceDir, &archive.TarOptions{
+		ExcludePatterns: excludes,
+		ChownOpts:       &idtools.Identity{UID: 0, GID: 0},
+	})
+	if err != nil {
+		return nil, "", fmt.Errorf("preparing build context: %w", err)
+	}
+
+	// NOTE: archive.CanonicalTarNameForPath normalizes path separators so the relative path will use /
+	// even on windows.
+	if strings.HasPrefix(relDockerfile, "../") {
+		dockerfileReader, err := os.Open(absDockerfile)
+		if err != nil {
+			return nil, "", fmt.Errorf("opening dockerfile: %w", err)
+		}
+		defer dockerfileReader.Close()
+		// dockerfile_path is outside of source_dir. we need to copy it inside the build context
+		// so that the docker engine can access it.
+		tar, relDockerfile, err = build.AddDockerfileToBuildContext(dockerfileReader, tar)
+		if err != nil {
+			return nil, "", fmt.Errorf("copying external dockerfile inside build context: %w", err)
+		}
+	}
+
+	return tar, relDockerfile, nil
+}
+
 // buildStaticSiteImage builds a container image that runs a webserver hosting the static site content
 func (b *DockerComponentBuilder) buildStaticSiteImage(ctx context.Context) error {
 	c, ok := b.component.(*godo.AppStaticSiteSpec)