diff --git a/go.mod b/go.mod index 30dc2938..b492c230 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.21.5 require ( github.com/aquasecurity/trivy v0.45.1 + github.com/containerd/containerd v1.7.13 github.com/cpuguy83/dockercfg v0.3.1 github.com/cpuguy83/go-docker v0.3.0 github.com/distribution/reference v0.6.0 @@ -57,7 +58,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/console v1.0.4 // indirect - github.com/containerd/containerd v1.7.13 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.2 // indirect diff --git a/integration/fixtures/test-images.json b/integration/fixtures/test-images.json index 7085bf87..b2ffd742 100644 --- a/integration/fixtures/test-images.json +++ b/integration/fixtures/test-images.json @@ -49,12 +49,20 @@ "description": "Custom dpkg/status.d with text names, no apt, libssl1.1", "ignoreErrors": false }, + { + "image": "registry.k8s.io/kube-proxy", + "tag": "v1.26.1", + "digest": "sha256:1e4f13f5f5c215813fb9c9c6f56da1c0354363f2a69bd12732658f79d585864f", + "distro": "Custom Google Distroless", + "description": "Custom dpkg/status.d with text names, with both libssl1.1 and libssl1", + "ignoreErrors": false + }, { "image": "docker.io/fluent/fluent-bit", "tag": "1.8.4", "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, no apt", + "description": "Custom dpkg/status.d with base64 names", "ignoreErrors": false }, { @@ -63,7 +71,7 @@ "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", "localName": "localimage:tag", "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, no apt, locally tagged with image name only", + "description": "Custom dpkg/status.d with base64 names, locally tagged with image name only", "ignoreErrors": false }, { diff --git a/integration/patch_test.go b/integration/patch_test.go index 5304febb..2776e81f 100644 --- a/integration/patch_test.go +++ b/integration/patch_test.go @@ -18,8 +18,13 @@ import ( "github.com/stretchr/testify/require" ) -//go:embed fixtures/trivy_ignore.rego -var trivyIgnore []byte +var ( + //go:embed fixtures/test-images.json + testImages []byte + + //go:embed fixtures/trivy_ignore.rego + trivyIgnore []byte +) type testImage struct { Image string `json:"image"` @@ -32,25 +37,8 @@ type testImage struct { } func TestPatch(t *testing.T) { - var file []byte - var err error - - // test distroless and non-distroless - if reportFile { - file, err = os.ReadFile("fixtures/test-images.json") - if err != nil { - t.Error("Unable to read test-images", err) - } - } else { - // only test non-distroless - file, err = os.ReadFile("fixtures/test-images-non-distroless.json") - if err != nil { - t.Error("Unable to read test-images", err) - } - } - var images []testImage - err = json.Unmarshal(file, &images) + err := json.Unmarshal(testImages, &images) require.NoError(t, err) tmp := t.TempDir() @@ -60,6 +48,10 @@ func TestPatch(t *testing.T) { for _, img := range images { img := img + if !reportFile { + img.IgnoreErrors = false + } + t.Run(img.Description, func(t *testing.T) { t.Parallel() diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 2e79eb6e..0555d694 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -4,6 +4,7 @@ import ( "bytes" "context" + "github.com/containerd/containerd/platforms" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" gwclient "github.com/moby/buildkit/frontend/gateway/client" @@ -60,7 +61,13 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri // Extracts the bytes of the file denoted by `path` from the state `st`. func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, path string) ([]byte, error) { - def, err := st.Marshal(ctx) + // since platform is obtained from host, override it in the case of Darwin + platform := platforms.Normalize(platforms.DefaultSpec()) + if platform.OS != "linux" { + platform.OS = "linux" + } + + def, err := st.Marshal(ctx, llb.Platform(platform)) if err != nil { return nil, err } diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index 42fcd365..91553a0a 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/containerd/containerd/platforms" "github.com/docker/buildx/build" "github.com/docker/cli/cli/config" log "github.com/sirupsen/logrus" @@ -19,6 +20,7 @@ import ( "github.com/distribution/reference" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" @@ -188,17 +190,21 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return nil, err } + osVersion, err := getOSVersion(ctx, fileBytes) + if err != nil { + ch <- err + return nil, err + } + // get package manager based on os family type - manager, err = pkgmgr.GetPackageManager(osType, config, workingFolder) + manager, err = pkgmgr.GetPackageManager(osType, osVersion, config, workingFolder) if err != nil { ch <- err return nil, err } - // do not specify updates, will update all - updates = nil } else { // get package manager based on os family type - manager, err = pkgmgr.GetPackageManager(updates.Metadata.OS.Type, config, workingFolder) + manager, err = pkgmgr.GetPackageManager(updates.Metadata.OS.Type, updates.Metadata.OS.Version, config, workingFolder) if err != nil { ch <- err return nil, err @@ -213,10 +219,15 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return nil, err } - def, err := patchedImageState.Marshal(ctx) + platform := platforms.Normalize(platforms.DefaultSpec()) + if platform.OS != "linux" { + platform.OS = "linux" + } + + def, err := patchedImageState.Marshal(ctx, llb.Platform(platform)) if err != nil { ch <- err - return nil, err + return nil, fmt.Errorf("unable to get platform from ImageState %w", err) } res, err := c.Solve(ctx, gwclient.SolveRequest{ @@ -321,6 +332,16 @@ func getOSType(ctx context.Context, osreleaseBytes []byte) (string, error) { } } +func getOSVersion(ctx context.Context, osreleaseBytes []byte) (string, error) { + r := bytes.NewReader(osreleaseBytes) + osData, err := osrelease.Parse(ctx, r) + if err != nil { + return "", fmt.Errorf("unable to parse os-release data %w", err) + } + + return osData["VERSION_ID"], nil +} + func dockerLoad(ctx context.Context, pipeR io.Reader) error { cmd := exec.CommandContext(ctx, "docker", "load") cmd.Stdin = pipeR diff --git a/pkg/patch/patch_test.go b/pkg/patch/patch_test.go index fc23d6e8..ad54c34b 100644 --- a/pkg/patch/patch_test.go +++ b/pkg/patch/patch_test.go @@ -203,3 +203,48 @@ func TestGetOSType(t *testing.T) { }) } } + +func TestGetOSVersion(t *testing.T) { + testCases := []struct { + osRelease []byte + errMsg string + expectedOSVersion string + }{ + { + osRelease: []byte(`PRETTY_NAME="Debian GNU/Linux 11 (bullseye)" + NAME="Debian GNU/Linux" + VERSION_ID="11" + VERSION="11 (bullseye)" + VERSION_CODENAME=bullseye + ID=debian + HOME_URL="https://www.debian.org/" + SUPPORT_URL="https://www.debian.org/support" + BUG_REPORT_URL="https://bugs.debian.org/" + `), + errMsg: "", + expectedOSVersion: "11", + }, + { + osRelease: []byte("Cannot Parse Version_ID"), + errMsg: "unable to parse os-release data osrelease: malformed line \"Cannot Parse Version_ID\"", + expectedOSVersion: "", + }, + } + + for _, tc := range testCases { + t.Run("TestGetOSVersion", func(t *testing.T) { + osVersion, err := getOSVersion(context.TODO(), tc.osRelease) + + var errMsg string + if err == nil { + errMsg = "" + } else { + errMsg = err.Error() + } + + // Use testify package to assert that the output manifest and error match the expected ones + assert.Equal(t, tc.expectedOSVersion, osVersion) + assert.Equal(t, tc.errMsg, errMsg) + }) + } +} diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index e0e56abc..2784c91e 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -4,8 +4,10 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "path/filepath" + "regexp" "strconv" "strings" @@ -22,6 +24,7 @@ const ( dpkgLibPath = "/var/lib/dpkg" dpkgStatusPath = dpkgLibPath + "/status" dpkgStatusFolder = dpkgLibPath + "/status.d" + dpkgDownloadPath = "/var/cache/apt/archives" statusdOutputFilename = "statusd_type" ) @@ -31,6 +34,8 @@ type dpkgManager struct { workingFolder string isDistroless bool statusdNames string + packageInfo map[string]string + osVersion string } type dpkgStatusType uint @@ -42,6 +47,7 @@ const ( DPKGStatusMixed DPKGStatusInvalid // must always be the last listed + Debian = "debian" ) func (st dpkgStatusType) String() string { @@ -72,15 +78,19 @@ func isLessThanDebianVersion(v1, v2 string) bool { } // Map the target image OSType & OSVersion to an appropriate tooling image. -func getAPTImageName(manifest *unversioned.UpdateManifest) string { - version := manifest.Metadata.OS.Version - if manifest.Metadata.OS.Type == "debian" { - version = strings.Split(manifest.Metadata.OS.Version, ".")[0] + "-slim" +func getAPTImageName(manifest *unversioned.UpdateManifest, osVersion string) string { + version := osVersion + osType := Debian + + if manifest == nil || manifest.Metadata.OS.Type == Debian { + version = strings.Split(version, ".")[0] + "-slim" + } else { + osType = manifest.Metadata.OS.Type } // TODO: support qualifying image name with designated repository - log.Debugf("Using %s:%s as basis for tooling image", manifest.Metadata.OS.Type, version) - return fmt.Sprintf("%s:%s", manifest.Metadata.OS.Type, version) + log.Debugf("Using %s:%s as basis for tooling image", osType, version) + return fmt.Sprintf("%s:%s", osType, version) } func getDPKGStatusType(b []byte) dpkgStatusType { @@ -103,8 +113,23 @@ func getDPKGStatusType(b []byte) dpkgStatusType { } func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned.UpdateManifest, ignoreErrors bool) (*llb.State, []string, error) { - // If manifest nil, update all packages (only for non-distroless right now) + // Probe for additional information to execute the appropriate update install graphs + toolImageName := getAPTImageName(manifest, dm.osVersion) + if err := dm.probeDPKGStatus(ctx, toolImageName, (manifest == nil)); err != nil { + return nil, nil, err + } + + // If manifest nil, update all packages if manifest == nil { + if dm.isDistroless { + updatedImageState, _, err := dm.unpackAndMergeUpdates(ctx, nil, toolImageName) + if err != nil { + return updatedImageState, nil, err + } + // add validation in the future + return updatedImageState, nil, nil + } + updatedImageState, _, err := dm.installUpdates(ctx, nil) if err != nil { return updatedImageState, nil, err @@ -113,6 +138,7 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned return updatedImageState, nil, nil } + // Else update according to specified updates // Validate and extract unique updates listed in input manifest debComparer := VersionComparer{isValidDebianVersion, isLessThanDebianVersion} updates, err := GetUniqueLatestUpdates(manifest.Updates, debComparer, ignoreErrors) @@ -124,12 +150,6 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned return &dm.config.ImageState, nil, nil } - // Probe for additional information to execute the appropriate update install graphs - toolImageName := getAPTImageName(manifest) - if err := dm.probeDPKGStatus(ctx, toolImageName); err != nil { - return nil, nil, err - } - var updatedImageState *llb.State var resultManifestBytes []byte if dm.isDistroless { @@ -156,7 +176,7 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned // Probe the target image for: // - DPKG status type to distinguish between regular and distroless images. // - Whether status.d contains base64-encoded package names. -func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) error { +func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, updateAll bool) error { imagePlatform, err := dm.config.ImageState.GetPlatform(ctx) if err != nil { return fmt.Errorf("unable to get image platform %w", err) @@ -196,6 +216,7 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er elif [ -d "$DPKG_STATUS_FOLDER" ]; then status="$DPKG_STATUS_IS_DIRECTORY" ls -1 "$DPKG_STATUS_FOLDER" > "$RESULT_STATUSD_PATH" + mv "$DPKG_STATUS_FOLDER"/* "$RESULTS_PATH" fi echo -n "$status" > "${RESULTS_PATH}/${STATUSD_OUTPUT_FILENAME}" `, @@ -215,8 +236,33 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er if err != nil { return err } + dm.statusdNames = strings.ReplaceAll(string(statusdNamesBytes), "\n", " ") dm.statusdNames = strings.TrimSpace(dm.statusdNames) + + // In the case of updating all packages, read each file to save package names and versions + if updateAll { + namesList := strings.Fields(dm.statusdNames) + packageInfo := make(map[string]string) + for _, name := range namesList { + fileBtyes, err := buildkit.ExtractFileFromState(ctx, dm.config.Client, &resultsState, name) + if err != nil { + return err + } + + if !strings.HasSuffix(name, ".md5sums") { + pkgName, pkgVersion, err := GetPackageInfo(string(fileBtyes)) + if err != nil { + return err + } + + packageInfo[pkgName] = pkgVersion + } + } + + dm.packageInfo = packageInfo + } + log.Infof("Processed status.d: %s", dm.statusdNames) dm.isDistroless = true return nil @@ -227,6 +273,29 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er } } +func GetPackageInfo(file string) (string, string, error) { + var packageName string + var packageVersion string + + packagePattern := regexp.MustCompile(`^Package:\s*(.*)`) + match := packagePattern.FindStringSubmatch(file) + if len(match) > 1 { + packageName = match[1] + } else { + return "", "", fmt.Errorf("no package name found for package") + } + + versionPattern := regexp.MustCompile(`Version:\s*(.*)`) + match = versionPattern.FindStringSubmatch(file) + if len(match) > 1 { + packageVersion = match[1] + } else { + return "", "", fmt.Errorf("no version found for package") + } + + return packageName, packageVersion, nil +} + // Patch a regular debian image with: // - sh and apt installed on the image // - valid dpkg status on the image @@ -303,21 +372,60 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers llb.IgnoreCache, ).Root() + // In the case of update all packages, only update packages that are not already latest version. Store these packages in packages.txt. + if updates == nil { + jsonPackageData, err := json.Marshal(dm.packageInfo) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal dm.packageInfo %w", err) + } + + updated = updated.Run( + llb.AddEnv("PACKAGES_PRESENT", string(jsonPackageData)), + llb.Args([]string{ + `bash`, `-c`, ` + json_str=$PACKAGES_PRESENT + update_packages="" + + while IFS=':' read -r package version; do + pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/') + pkg_version=$(echo "$version" | sed 's/^"\(.*\)"$/\1/') + latest_version=$(apt show $pkg_name 2>/dev/null | awk -F ': ' '/Version:/{print $2}') + + if [ "$latest_version" != "$pkg_version" ]; then + update_packages="$update_packages $pkg_name" + fi + done <<< "$(echo "$json_str" | tr -d '{}\n' | tr ',' '\n')" + + mkdir /var/cache/apt/archives + cd /var/cache/apt/archives + echo "$update_packages" > packages.txt + `, + })).Root() + } + // Download all requested update packages without specifying the version. This works around: // - Reports being slightly out of date, where a newer security revision has displaced the one specified leading to not found errors. // - Reports not specifying version epochs correct (e.g. bsdutils=2.36.1-8+deb11u1 instead of with epoch as 1:2.36.1-8+dev11u1) - const aptDownloadTemplate = "apt download --no-install-recommends %s" + var downloadCmd string pkgStrings := []string{} - for _, u := range updates { - pkgStrings = append(pkgStrings, u.Name) + if updates != nil { + aptDownloadTemplate := "apt download --no-install-recommends %s" + for _, u := range updates { + pkgStrings = append(pkgStrings, u.Name) + } + downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) + } else { + // only updated the outdated pacakges from packages.txt + downloadCmd = "xargs -a packages.txt -n 1 apt download --no-install-recommends" } - downloadCmd := fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) - downloaded := updated.Dir(downloadPath).Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() + + downloaded := updated.Dir(dpkgDownloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() + diffState := llb.Diff(updated, downloaded) // Scripted enumeration and dpkg unpack of all downloaded packages [layer to merge with target] const extractTemplate = `find %s -name '*.deb' -exec dpkg-deb -x '{}' %s \;` - extractCmd := fmt.Sprintf(extractTemplate, downloadPath, unpackPath) - unpacked := downloaded.Run(llb.Shlex(extractCmd)).Root() + extractCmd := fmt.Sprintf(extractTemplate, dpkgDownloadPath, unpackPath) + unpacked := downloaded.Run(llb.AddMount(dpkgDownloadPath, diffState), llb.Shlex(extractCmd)).Root() unpackedToRoot := llb.Scratch().File(llb.Copy(unpacked, unpackPath, "/", &llb.CopyInfo{CopyDirContentsOnly: true})) // Scripted extraction of all debinfo for version checking to separate layer into local mount @@ -325,7 +433,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers mkFolders := downloaded.File(llb.Mkdir(resultsPath, 0o744, llb.WithParents(true))).File(llb.Mkdir(dpkgStatusFolder, 0o744, llb.WithParents(true))) const writeFieldsTemplate = `find . -name '*.deb' -exec sh -c "dpkg-deb -f {} > %s" \;` writeFieldsCmd := fmt.Sprintf(writeFieldsTemplate, filepath.Join(resultsPath, "{}.fields")) - fieldsWritten := mkFolders.Dir(downloadPath).Run(llb.Shlex(writeFieldsCmd)).Root() + fieldsWritten := mkFolders.Dir(dpkgDownloadPath).Run(llb.Shlex(writeFieldsCmd)).Root() // Write the name and version of the packages applied to the results.manifest file for the host const outputResultsTemplate = `find . -name '*.fields' -exec sh -c 'grep "^Package:\|^Version:" {} >> %s' \;` diff --git a/pkg/pkgmgr/dpkg_test.go b/pkg/pkgmgr/dpkg_test.go index 68052e5e..64c0037b 100644 --- a/pkg/pkgmgr/dpkg_test.go +++ b/pkg/pkgmgr/dpkg_test.go @@ -125,7 +125,7 @@ func TestGetAPTImageName(t *testing.T) { // Run test cases for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := getAPTImageName(tc.manifest) + got := getAPTImageName(tc.manifest, tc.manifest.Metadata.OS.Version) if got != tc.want { t.Errorf("getAPTImageName() = %v, want %v", got, tc.want) } @@ -339,7 +339,7 @@ func TestValidateDebianPackageVersions(t *testing.T) { } } -func Test_dpkgManager_GetPackageType(t *testing.T) { +func TestGetPackageType(t *testing.T) { type fields struct { config *buildkit.Config workingFolder string @@ -376,3 +376,66 @@ func Test_dpkgManager_GetPackageType(t *testing.T) { }) } } + +func Test_GetPackageInfo(t *testing.T) { + type fields struct { + name string + version string + errMsg string + } + tests := []struct { + name string + file string + want fields + }{ + { + name: "valid package file format", + file: `Package: tzdata + Version: 2021a-1+deb11u8 + Architecture: all + Maintainer: GNU Libc Maintainers + Installed-Size: 3393 + Depends: debconf (>= 0.5) | debconf-2.0 + Provides: tzdata-bullseye + Section: localization + Priority: required + Multi-Arch: foreign + Homepage: https://www.iana.org/time-zones + Description: time zone and daylight-saving time data + This package contains data required for the implementation of + standard local time for many representative locations around the + globe. It is updated periodically to reflect changes made by + political bodies to time zone boundaries, UTC offsets, and + daylight-saving rules.`, + want: fields{ + name: "tzdata", + version: "2021a-1+deb11u8", + errMsg: "", + }, + }, + { + name: "invalid package file format", + file: "PackageVersion", + want: fields{ + name: "", + version: "", + errMsg: "no package name found for package", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name, version, err := GetPackageInfo(tt.file) + var errMsg string + if err == nil { + errMsg = "" + } else { + errMsg = err.Error() + } + + if name != tt.want.name || version != tt.want.version || errMsg != tt.want.errMsg { + t.Errorf("GetPackageInfo() = Name: %v, Version: %v Error: %v, want Name: %v, Version: %v, Error: %v", name, version, err, tt.want.name, tt.want.version, tt.want.errMsg) + } + }) + } +} diff --git a/pkg/pkgmgr/pkgmgr.go b/pkg/pkgmgr/pkgmgr.go index e7d6856e..d2966efb 100644 --- a/pkg/pkgmgr/pkgmgr.go +++ b/pkg/pkgmgr/pkgmgr.go @@ -27,12 +27,12 @@ type PackageManager interface { GetPackageType() string } -func GetPackageManager(osType string, config *buildkit.Config, workingFolder string) (PackageManager, error) { +func GetPackageManager(osType string, osVersion string, config *buildkit.Config, workingFolder string) (PackageManager, error) { switch osType { case "alpine": return &apkManager{config: config, workingFolder: workingFolder}, nil case "debian", "ubuntu": - return &dpkgManager{config: config, workingFolder: workingFolder}, nil + return &dpkgManager{config: config, workingFolder: workingFolder, osVersion: osVersion}, nil case "cbl-mariner", "centos", "redhat", "rocky", "amazon": return &rpmManager{config: config, workingFolder: workingFolder}, nil default: diff --git a/pkg/pkgmgr/pkgmgr_test.go b/pkg/pkgmgr/pkgmgr_test.go index d72b0bad..03229a41 100644 --- a/pkg/pkgmgr/pkgmgr_test.go +++ b/pkg/pkgmgr/pkgmgr_test.go @@ -17,7 +17,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an apkManager for alpine", func(t *testing.T) { // Call the GetPackageManager function with "alpine" as osType - manager, err := GetPackageManager("alpine", config, workingFolder) + manager, err := GetPackageManager("alpine", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -29,7 +29,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return a dpkgManager for debian", func(t *testing.T) { // Call the GetPackageManager function with "debian" as osType - manager, err := GetPackageManager("debian", config, workingFolder) + manager, err := GetPackageManager("debian", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -41,7 +41,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return a dpkgManager for ubuntu", func(t *testing.T) { // Call the GetPackageManager function with "ubuntu" as osType - manager, err := GetPackageManager("ubuntu", config, workingFolder) + manager, err := GetPackageManager("ubuntu", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -53,7 +53,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an rpmManager for cbl-mariner", func(t *testing.T) { // Call the GetPackageManager function with "cbl-mariner" as osType - manager, err := GetPackageManager("cbl-mariner", config, workingFolder) + manager, err := GetPackageManager("cbl-mariner", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -65,7 +65,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an rpmManager for redhat", func(t *testing.T) { // Call the GetPackageManager function with "redhat" as osType - manager, err := GetPackageManager("redhat", config, workingFolder) + manager, err := GetPackageManager("redhat", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -77,7 +77,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an error for unsupported osType", func(t *testing.T) { // Call the GetPackageManager function with "unsupported" as osType - manager, err := GetPackageManager("unsupported", config, workingFolder) + manager, err := GetPackageManager("unsupported", "", config, workingFolder) // Assert that there is an error and the manager is nil assert.Error(t, err) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index fee883d1..19a91442 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "path/filepath" @@ -43,6 +44,7 @@ type rpmManager struct { workingFolder string rpmTools rpmToolPaths isDistroless bool + packageInfo map[string]string } type rpmDBType uint @@ -286,6 +288,16 @@ func (rm *rpmManager) probeRPMStatus(ctx context.Context, toolImage string) erro switch rpmDB { case RPMDBManifests: rm.isDistroless = true + rpmManifest2File, err := buildkit.ExtractFileFromState(ctx, rm.config.Client, &outState, rpmManifest2) + if err != nil { + return err + } + // parse container-manifest-2 to get installed package names and versions + pkgInfo, err := parseManifestFile(string(rpmManifest2File)) + if err != nil { + return err + } + rm.packageInfo = pkgInfo case RPMDBNone, RPMDBMixed: err := fmt.Errorf("could not find determine RPM DB type of target image: %v", rpmDB) log.Error(err) @@ -326,6 +338,30 @@ func (rm *rpmManager) probeRPMStatus(ctx context.Context, toolImage string) erro return nil } +func parseManifestFile(file string) (map[string]string, error) { + // split into lines + file = strings.TrimSuffix(file, "\n") + lines := strings.Split(file, "\n") + + resultMap := make(map[string]string) + + // iterate over lines + for _, line := range lines { + // split line into columns + columns := strings.Split(line, "\t") + + if len(columns) >= 2 { + // get package name and version + name := columns[0] + version := strings.TrimSuffix(columns[1], ".cm2") + resultMap[name] = version + } else { + return nil, errors.New("unexpected format when parsing rpm manifest file") + } + } + return resultMap, nil +} + // Patch a regular RPM-based image with: // - sh and an appropriate tool installed on the image (yum, dnf, microdnf) // - valid rpm database on the image @@ -394,16 +430,53 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi toolsInstalled := toolingBase.Run(llb.Shlex(installToolsCmd), llb.WithProxy(utils.GetProxy())).Root() busyboxCopied := toolsInstalled.Dir(downloadPath).Run(llb.Shlex("cp /usr/sbin/busybox .")).Root() + // In the case of update all packages, only update packages that are not latest version. Store these packages in packages.txt. + if updates == nil { + jsonPackageData, err := json.Marshal(rm.packageInfo) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal dm.packageInfo %w", err) + } + + busyboxCopied = busyboxCopied.Run( + llb.AddEnv("PACKAGES_PRESENT", string(jsonPackageData)), + llb.Args([]string{ + `bash`, `-c`, ` + json_str=$PACKAGES_PRESENT + update_packages="" + + while IFS=':' read -r package version; do + pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/') + + pkg_version=$(echo "$version" | sed 's/^"\(.*\)"$/\1/') + latest_version=$(yum info $pkg_name 2>/dev/null | grep "Version" | sed -n '$s/Version *: //p') + + if [ "$latest_version" != "$pkg_version" ]; then + update_packages="$update_packages $pkg_name" + fi + done <<< "$(echo "$json_str" | tr -d '{}\n' | tr ',' '\n')" + + echo "$update_packages" > packages.txt + `, + })).Root() + } + // Download all requested update packages without specifying the version. This works around: // - Reports being slightly out of date, where a newer security revision has displaced the one specified leading to not found errors. // - Reports not specifying version epochs correct (e.g. bsdutils=2.36.1-8+deb11u1 instead of with epoch as 1:2.36.1-8+dev11u1) // - Reports specifying remediation packages for cbl-mariner v1 instead of v2 (e.g. *.cm1.aarch64 instead of *.cm2.aarch64) - const rpmDownloadTemplate = `yumdownloader --downloadonly --downloaddir=. --best -y %s` - pkgStrings := []string{} - for _, u := range updates { - pkgStrings = append(pkgStrings, u.Name) + var downloadCmd string + if updates != nil { + const rpmDownloadTemplate = `yumdownloader --downloadonly --downloaddir=. --best -y %s` + pkgStrings := []string{} + for _, u := range updates { + pkgStrings = append(pkgStrings, u.Name) + } + downloadCmd = fmt.Sprintf(rpmDownloadTemplate, strings.Join(pkgStrings, " ")) + } else { + // only updated the outdated pacakges from packages.txt + downloadCmd = `xargs -a packages.txt -n 1 yumdownloader --downloadonly --downloaddir=. --best -y` } - downloadCmd := fmt.Sprintf(rpmDownloadTemplate, strings.Join(pkgStrings, " ")) + downloaded := busyboxCopied.Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() // Scripted enumeration and rpm install of all downloaded packages under the download folder as root diff --git a/pkg/vex/openvex_test.go b/pkg/vex/openvex_test.go index a7fdedef..92716719 100644 --- a/pkg/vex/openvex_test.go +++ b/pkg/vex/openvex_test.go @@ -12,8 +12,8 @@ import ( func TestOpenVex_CreateVEXDocument(t *testing.T) { config := &buildkit.Config{} workingFolder := "/tmp" - alpineManager, _ := pkgmgr.GetPackageManager("alpine", config, workingFolder) - debianManager, _ := pkgmgr.GetPackageManager("debian", config, workingFolder) + alpineManager, _ := pkgmgr.GetPackageManager("alpine", "", config, workingFolder) + debianManager, _ := pkgmgr.GetPackageManager("debian", "", config, workingFolder) patchedImageName := "foo.io/bar:latest" t.Setenv("COPA_VEX_AUTHOR", "test author") diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index 6baef3e9..04e8717c 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -11,7 +11,7 @@ import ( func TestTryOutputVexDocument(t *testing.T) { config := &buildkit.Config{} workingFolder := "/tmp" - alpineManager, _ := pkgmgr.GetPackageManager("alpine", config, workingFolder) + alpineManager, _ := pkgmgr.GetPackageManager("alpine", "", config, workingFolder) patchedImageName := "patched" type args struct { diff --git a/website/docs/quick-start.md b/website/docs/quick-start.md index fb76e10e..72e399f7 100644 --- a/website/docs/quick-start.md +++ b/website/docs/quick-start.md @@ -16,7 +16,15 @@ This sample illustrates how to patch containers using vulnerability reports with * Alternatively, see [scanner plugins](#scanner-plugins) for custom scanner support. ## Sample Steps +Copa can patch images in two ways: +- Update only vulnerable packages as detected by a supported scanner report. +- Update all outdated packages in a container, regardless of vulnerability status. +:::note +The update all functionality allows you to address discrepancies that may arise between scanners and the packages they flag as vulnerable. It is important to note, however, that some upgrades can introduce dependency or compatibility conflicts. +::: + +### Patch with scanner report: 1. Scan the container image for patchable OS vulnerabilities, outputting the results to a JSON file: ```bash @@ -102,5 +110,30 @@ This sample illustrates how to patch containers using vulnerability reports with 2024/01/22 23:32:54 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576 2024/01/22 23:32:54 [notice] 1#1: start worker processes ``` + You can stop the container by opening a new shell instance and running: `docker stop nginx-test` + +### Patch all outdated packages: +1. Run Copa with a buildkit connection as described above, and omit the report flag: + ```bash + copa patch -i docker.io/library/nginx:1.21.6 + ``` +2. Run the container to verify that the image has no regressions: + ```bash + $ docker run -it --rm --name nginx-test docker.io/library/nginx:1.21.6-patched + /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration + /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ + /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh + 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf + 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf + /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh + /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh + /docker-entrypoint.sh: Configuration complete; ready for start up + 2024/01/22 23:32:54 [notice] 1#1: using the "epoll" event method + 2024/01/22 23:32:54 [notice] 1#1: nginx/1.21.6 + 2024/01/22 23:32:54 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6) + 2024/01/22 23:32:54 [notice] 1#1: OS: Linux 6.2.0-1018-azure + 2024/01/22 23:32:54 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576 + 2024/01/22 23:32:54 [notice] 1#1: start worker processes + ``` You can stop the container by opening a new shell instance and running: `docker stop nginx-test`