Merge branch 'main' into dependabot/npm_and_yarn/website/all-3ca6734895

.github/workflows/release-docs.yml

@@ -34,7 +34,7 @@ jobs:
         run: make version-docs
 
       - name: Create release pull request
-        uses: peter-evans/create-pull-request@6d6857d36972b65feb161a90e484f2984215f83e # v6.0.5
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
         with:
           commit-message: "chore: Generate ${{ env.NEWVERSION }} docs"
           title: "chore: Generate ${{ env.NEWVERSION }} docs"

Makefile

@@ -43,7 +43,7 @@ endif
 # Build output variables
 CLI_BINARY        := copa
 OUT_DIR           := ./dist
-BINS_OUT_DIR      := $(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)
+BINS_OUT_DIR      ?= $(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)
 
 ################################################################################
 # Target: build (default action)                                               #

go.mod

@@ -10,14 +10,14 @@ require (
 	github.com/cpuguy83/dockercfg v0.3.1
 	github.com/cpuguy83/go-docker v0.3.0
 	github.com/distribution/reference v0.6.0
-	github.com/docker/buildx v0.15.0
+	github.com/docker/buildx v0.15.1
 	github.com/docker/cli v26.1.4+incompatible
-	github.com/google/go-containerregistry v0.19.1
+	github.com/google/go-containerregistry v0.19.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
 	github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422
 	github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075
-	github.com/moby/buildkit v0.14.0
+	github.com/moby/buildkit v0.14.1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0
 	github.com/openvex/go-vex v0.2.5

go.sum

@@ -112,8 +112,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
 github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/buildx v0.15.0 h1:PVq4IMnTvw1Sx0RKDWbfi2eTGawFd9CMBYnz9xat93Y=
-github.com/docker/buildx v0.15.0/go.mod h1:AdkB1RIcU4rfZ6mpw2PA2pOi1ppI9yvFXkVEpq5EmS4=
+github.com/docker/buildx v0.15.1 h1:1cO6JIc0rOoC8tlxfXoh1HH1uxaNvYH1q7J7kv5enhw=
+github.com/docker/buildx v0.15.1/go.mod h1:16DQgJqoggmadc1UhLaUTPqKtR+PlByN/kyXFdkhFCo=
 github.com/docker/cli v26.1.4+incompatible h1:I8PHdc0MtxEADqYJZvhBrW9bo8gawKwwenxRM7/rLu8=
 github.com/docker/cli v26.1.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -200,8 +200,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY=
-github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
+github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w=
+github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -307,8 +307,8 @@ github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/moby/buildkit v0.14.0 h1:mHv2lFS8znLDRc4SMyM2B9tPjxWh2blMvr0H7ARquNM=
-github.com/moby/buildkit v0.14.0/go.mod h1:1XssG7cAqv5Bz1xcGMxJL123iCv5TYN4Z/qf647gfuk=
+github.com/moby/buildkit v0.14.1 h1:2epLCZTkn4CikdImtsLtIa++7DzCimrrZCT1sway+oI=
+github.com/moby/buildkit v0.14.1/go.mod h1:1XssG7cAqv5Bz1xcGMxJL123iCv5TYN4Z/qf647gfuk=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=

pkg/buildkit/buildkit.go

@@ -90,6 +90,10 @@ func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State,
 	})
 }
 
+func Sh(cmd string) llb.RunOption {
+	return llb.Args([]string{"/bin/sh", "-c", cmd})
+}
+
 func ArrayFile(input []string) []byte {
 	var b bytes.Buffer
 	for _, s := range input {

pkg/pkgmgr/apk.go

@@ -120,7 +120,7 @@ func validateAPKPackageVersions(updates unversioned.UpdatePackages, cmp VersionC
 func (am *apkManager) InstallUpdates(ctx context.Context, manifest *unversioned.UpdateManifest, ignoreErrors bool) (*llb.State, []string, error) {
 	// If manifest is nil, update all packages
 	if manifest == nil {
-		updatedImageState, _, err := am.upgradePackages(ctx, nil)
+		updatedImageState, _, err := am.upgradePackages(ctx, nil, ignoreErrors)
 		if err != nil {
 			return updatedImageState, nil, err
 		}
@@ -140,7 +140,7 @@ func (am *apkManager) InstallUpdates(ctx context.Context, manifest *unversioned.
 	}
 	log.Debugf("latest unique APKs: %v", updates)
 
-	updatedImageState, resultsBytes, err := am.upgradePackages(ctx, updates)
+	updatedImageState, resultsBytes, err := am.upgradePackages(ctx, updates, ignoreErrors)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -161,7 +161,7 @@ func (am *apkManager) InstallUpdates(ctx context.Context, manifest *unversioned.
 // TODO: support "distroless" Alpine images (e.g. APKO images)
 // Still assumes that APK exists in the target image and is pathed, which can be addressed by
 // mounting a copy of apk-tools-static into the image and invoking apk-static directly.
-func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.UpdatePackages) (*llb.State, []byte, error) {
+func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) {
 	// TODO: Add support for custom APK config
 	apkUpdated := am.config.ImageState.Run(llb.Shlex("apk update"), llb.WithProxy(utils.GetProxy()), llb.IgnoreCache).Root()
 
@@ -200,8 +200,13 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U
 		}
 	} else {
 		// if updates is not specified, update all packages
-		installCmd := `apk upgrade --no-cache`
-		apkInstalled = apkUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root()
+		installCmd := `output=$(apk upgrade --no-cache 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi`
+		apkInstalled = apkUpdated.Run(buildkit.Sh(installCmd), llb.WithProxy(utils.GetProxy())).Root()
+
+		// Validate no errors were encountered if updating all
+		if !ignoreErrors {
+			apkInstalled = apkInstalled.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
+		}
 	}
 
 	// Diff the installed updates and merge that into the target image

pkg/pkgmgr/dpkg.go

@@ -122,19 +122,17 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned
 	// If manifest nil, update all packages
 	if manifest == nil {
 		if dm.isDistroless {
-			updatedImageState, _, err := dm.unpackAndMergeUpdates(ctx, nil, toolImageName)
+			updatedImageState, _, err := dm.unpackAndMergeUpdates(ctx, nil, toolImageName, ignoreErrors)
 			if err != nil {
 				return updatedImageState, nil, err
 			}
-			// add validation in the future
 			return updatedImageState, nil, nil
 		}
 
-		updatedImageState, _, err := dm.installUpdates(ctx, nil)
+		updatedImageState, _, err := dm.installUpdates(ctx, nil, ignoreErrors)
 		if err != nil {
 			return updatedImageState, nil, err
 		}
-		// add validation in the future
 		return updatedImageState, nil, nil
 	}
 
@@ -153,12 +151,12 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned
 	var updatedImageState *llb.State
 	var resultManifestBytes []byte
 	if dm.isDistroless {
-		updatedImageState, resultManifestBytes, err = dm.unpackAndMergeUpdates(ctx, updates, toolImageName)
+		updatedImageState, resultManifestBytes, err = dm.unpackAndMergeUpdates(ctx, updates, toolImageName, ignoreErrors)
 		if err != nil {
 			return nil, nil, err
 		}
 	} else {
-		updatedImageState, resultManifestBytes, err = dm.installUpdates(ctx, updates)
+		updatedImageState, resultManifestBytes, err = dm.installUpdates(ctx, updates, ignoreErrors)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -304,7 +302,7 @@ func GetPackageInfo(file string) (string, string, error) {
 //
 // TODO: Support Debian images with valid dpkg status but missing tools. No current examples exist in test set
 // i.e. extra RunOption to mount a copy of busybox-static or full apt install into the image and invoking that.
-func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages) (*llb.State, []byte, error) {
+func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) {
 	// TODO: Add support for custom APT config and gpg key injection
 	// Since this takes place in the target container, it can interfere with install actions
 	// such as the installation of the updated debian-archive-keyring package, so it's probably best
@@ -330,11 +328,16 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U
 		installCmd = fmt.Sprintf(aptInstallTemplate, strings.Join(pkgStrings, " "))
 	} else {
 		// if updates is not specified, update all packages
-		installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove"`
+		installCmd = `sh -c "output=$(apt upgrade -y && apt clean -y && apt autoremove 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi"`
 	}
 
 	aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root()
 
+	// Validate no errors were encountered if updating all
+	if updates == nil && !ignoreErrors {
+		aptInstalled = aptInstalled.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
+	}
+
 	// Write results.manifest to host for post-patch validation
 	const outputResultsTemplate = `sh -c 'grep "^Package:\|^Version:" "%s" >> "%s"'`
 	outputResultsCmd := fmt.Sprintf(outputResultsTemplate, dpkgStatusPath, resultManifest)
@@ -352,7 +355,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U
 	return &patchMerge, resultsBytes, nil
 }
 
-func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string) (*llb.State, []byte, error) {
+func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string, ignoreErrors bool) (*llb.State, []byte, error) {
 	imagePlatform, err := dm.config.ImageState.GetPlatform(ctx)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to get image platform %w", err)
@@ -415,11 +418,25 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers
 		}
 		downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " "))
 	} else {
-		// only updated the outdated pacakges from packages.txt
-		downloadCmd = "xargs -a packages.txt -n 1 apt download --no-install-recommends"
+		// only update the outdated pacakges from packages.txt
+		downloadCmd = `
+		packages=$(<packages.txt)
+		for package in $packages; do
+			output=$(apt download --no-install-recommends "$package" 2>&1)
+			if [ $? -ne 0 ]; then
+				echo "$output" >>error_log.txt
+			fi
+		done
+		`
 	}
 
 	downloaded := updated.Dir(dpkgDownloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root()
+
+	// Validate no errors were encountered if updating all
+	if updates == nil && !ignoreErrors {
+		downloaded = downloaded.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
+	}
+
 	diffState := llb.Diff(updated, downloaded)
 
 	// Scripted enumeration and dpkg unpack of all downloaded packages [layer to merge with target]

pkg/pkgmgr/rpm.go

@@ -211,12 +211,12 @@ func (rm *rpmManager) InstallUpdates(ctx context.Context, manifest *unversioned.
 	var updatedImageState *llb.State
 	var resultManifestBytes []byte
 	if rm.isDistroless {
-		updatedImageState, resultManifestBytes, err = rm.unpackAndMergeUpdates(ctx, updates, toolImageName)
+		updatedImageState, resultManifestBytes, err = rm.unpackAndMergeUpdates(ctx, updates, toolImageName, ignoreErrors)
 		if err != nil {
 			return nil, nil, err
 		}
 	} else {
-		updatedImageState, resultManifestBytes, err = rm.installUpdates(ctx, updates)
+		updatedImageState, resultManifestBytes, err = rm.installUpdates(ctx, updates, ignoreErrors)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -383,7 +383,7 @@ func parseManifestFile(file string) (map[string]string, error) {
 //
 // TODO: Support RPM-based images with valid rpm status but missing tools. (e.g. calico images > v3.21.0)
 // i.e. extra RunOption to mount a copy of rpm tools installed into the image and invoking that.
-func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages) (*llb.State, []byte, error) {
+func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) {
 	pkgs := ""
 
 	// If specific updates, provided, parse into pkg names, else will update all
@@ -400,20 +400,25 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up
 	var installCmd string
 	switch {
 	case rm.rpmTools["dnf"] != "":
-		const dnfInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all'`
+		const dnfInstallTemplate = `sh -c 'output=$(%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'`
 		installCmd = fmt.Sprintf(dnfInstallTemplate, rm.rpmTools["dnf"], pkgs)
 	case rm.rpmTools["yum"] != "":
-		const yumInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all'`
+		const yumInstallTemplate = `sh -c 'output=$(%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'`
 		installCmd = fmt.Sprintf(yumInstallTemplate, rm.rpmTools["yum"], pkgs)
 	case rm.rpmTools["microdnf"] != "":
-		const microdnfInstallTemplate = `sh -c '%[1]s update %[2]s && %[1]s clean all'`
+		const microdnfInstallTemplate = `sh -c 'output=$(%[1]s update %[2]s && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'`
 		installCmd = fmt.Sprintf(microdnfInstallTemplate, rm.rpmTools["microdnf"], pkgs)
 	default:
 		err := errors.New("unexpected: no package manager tools were found for patching")
 		return nil, nil, err
 	}
 	installed := rm.config.ImageState.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root()
 
+	// Validate no errors were encountered if updating all
+	if updates == nil && !ignoreErrors {
+		installed = installed.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
+	}
+
 	// Write results.manifest to host for post-patch validation
 	var resultBytes []byte
 	var err error
@@ -434,7 +439,7 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up
 	return &patchMerge, resultBytes, nil
 }
 
-func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string) (*llb.State, []byte, error) {
+func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string, ignoreErrors bool) (*llb.State, []byte, error) {
 	// Spin up a build tooling container to fetch and unpack packages to create patch layer.
 	// Pull family:version -> need to create version to base image map
 	toolingBase := llb.Image(toolImage,
@@ -489,10 +494,23 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi
 		downloadCmd = fmt.Sprintf(rpmDownloadTemplate, strings.Join(pkgStrings, " "))
 	} else {
 		// only updated the outdated pacakges from packages.txt
-		downloadCmd = `xargs -a packages.txt -n 1 yumdownloader --downloadonly --downloaddir=. --best -y`
+		downloadCmd = `
+		packages=$(<packages.txt)
+		for package in $packages; do
+			output=$(yumdownloader --downloadonly --downloaddir=. --best -y "$package" 2>&1)
+			if [ $? -ne 0 ]; then
+				echo "$output" >>error_log.txt
+			fi
+		done
+		`
 	}
 
-	downloaded := busyboxCopied.Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root()
+	downloaded := busyboxCopied.Run(buildkit.Sh(downloadCmd), llb.WithProxy(utils.GetProxy())).Root()
+
+	// Validate no errors were encountered if updating all
+	if updates == nil && !ignoreErrors {
+		downloaded = downloaded.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root()
+	}
 
 	// Scripted enumeration and rpm install of all downloaded packages under the download folder as root
 	const extractTemplate = `sh -c 'for f in %[1]s/*.rpm ; do rpm2cpio "$f" | cpio -idmv -D %[1]s ; done'`