From 85a4e8a17f01719048ccafb71d19985c97b12cef Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani <45800463+rchincha@users.noreply.github.com> Date: Tue, 16 Jan 2024 23:37:26 -0800 Subject: [PATCH] fix: make the spdx output more complete (#53) sha1 checksums are commonly checked by existing scanners/tools. Some of the fields are not set, so set them. Signed-off-by: Ramkumar Chinchani --- Makefile | 2 +- pkg/distro/apk/apk.go | 51 ++++++++++++++++++++++++++--------- pkg/distro/deb/deb.go | 45 ++++++++++++++++++++++++------- pkg/distro/rpm/rpm.go | 62 ++++++++++++++++++++++++++++++++++--------- test/bom.bats | 2 +- 5 files changed, 125 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index d3ce6d5..97a3eae 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ GOLINTER_VERSION := v1.52.2 # OCI registry ZOT := $(TOOLSDIR)/bin/zot -ZOT_VERSION := 2.0.0-rc5 +ZOT_VERSION := 2.0.0 # OCI registry clients ORAS := $(TOOLSDIR)/bin/oras ORAS_VERSION := 1.0.0-rc.1 diff --git a/pkg/distro/apk/apk.go b/pkg/distro/apk/apk.go index 07ef519..d3c5a96 100644 --- a/pkg/distro/apk/apk.go +++ b/pkg/distro/apk/apk.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bufio" "compress/gzip" + "crypto/sha1" //nolint:gosec // used only to produce the sha1 checksum field "crypto/sha256" "encoding/base64" "encoding/hex" @@ -93,6 +94,7 @@ func ParsePackage(input, output, author, organization, license string) error { }{ Person: apk.PkgInfo.PkgMaintainer, }, + FilesAnalyzed: true, LicenseDeclared: pkglicense, } @@ -151,18 +153,23 @@ func ParsePackage(input, output, author, organization, license string) error { } } - cksum := sha256.Sum256(buf) + cksumSHA1 := sha1.Sum(buf) //nolint:gosec // used only to produce the sha1 checksum field + cksumSHA256 := sha256.Sum256(buf) log.Info().Str("name", hdr.Name). Int("size", bufsz). - Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksum[:]))). + Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksumSHA256[:]))). Msg("file entry detected") sfile := &spdx.File{ Entity: spdx.Entity{ - Name: fmt.Sprintf("/%s", hdr.Name), - Checksum: map[string]string{"SHA256": hex.EncodeToString(cksum[:])}, + Name: fmt.Sprintf("/%s", hdr.Name), + Checksum: map[string]string{ + "SHA1": hex.EncodeToString(cksumSHA1[:]), + "SHA256": hex.EncodeToString(cksumSHA256[:]), + }, }, + LicenseInfoInFile: license, } if err := spkg.AddFile(sfile); err != nil { log.Error().Err(err).Msg("unable to add file to package") @@ -192,11 +199,12 @@ func InstalledPackage(doc *spdx.Document, pkg *IndexEntry, files []string) error }{ Person: pkg.PackageMaintainer, }, + FilesAnalyzed: true, LicenseDeclared: pkg.PackageLicense, } for _, file := range files { - sinfo, err := os.Stat(file) + info, err := os.Stat(file) if err != nil { if errors.Is(err, os.ErrNotExist) { continue @@ -205,8 +213,8 @@ func InstalledPackage(doc *spdx.Document, pkg *IndexEntry, files []string) error return err } - if !sinfo.Mode().IsRegular() { - log.Warn().Str("file", file).Interface("mode", sinfo.Mode()).Msg("skipping entry since not a regular file") + if !info.Mode().IsRegular() { + log.Warn().Str("file", file).Interface("mode", info.Mode()).Msg("skipping entry since not a regular file") continue } @@ -217,18 +225,35 @@ func InstalledPackage(doc *spdx.Document, pkg *IndexEntry, files []string) error } defer fhandle.Close() - shaWriter := sha256.New() - if _, err := io.Copy(shaWriter, fhandle); err != nil { - return err + buf := make([]byte, info.Size()) + + var bufsz int + + if bufsz, err = fhandle.Read(buf); err != nil { + if !errors.Is(err, io.EOF) { + log.Error().Err(err).Str("name", info.Name()).Msg("unable to read content") + + return err + } } - cksum := shaWriter.Sum(nil) + cksumSHA1 := sha1.Sum(buf) //nolint:gosec // used only to produce the sha1 checksum field + cksumSHA256 := sha256.Sum256(buf) + + log.Info().Str("name", info.Name()). + Int("size", bufsz). + Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksumSHA256[:]))). + Msg("file entry detected") sfile := spdx.NewFile() + sfile.LicenseInfoInFile = "unknown" sfile.SetEntity( &spdx.Entity{ - Name: file, - Checksum: map[string]string{"SHA256": hex.EncodeToString(cksum)}, + Name: file, + Checksum: map[string]string{ + "SHA1": hex.EncodeToString(cksumSHA1[:]), + "SHA256": hex.EncodeToString(cksumSHA256[:]), + }, }, ) diff --git a/pkg/distro/deb/deb.go b/pkg/distro/deb/deb.go index eeb76f2..bf99dd1 100644 --- a/pkg/distro/deb/deb.go +++ b/pkg/distro/deb/deb.go @@ -3,6 +3,7 @@ package deb import ( "archive/tar" "bufio" + "crypto/sha1" //nolint:gosec // used only to produce the sha1 checksum field "encoding/hex" "errors" "fmt" @@ -48,6 +49,7 @@ func ParsePackage(input, output, author, organization, license string) error { }{ Person: debfile.Control.Maintainer, }, + FilesAnalyzed: true, LicenseDeclared: license, } @@ -87,18 +89,23 @@ func ParsePackage(input, output, author, organization, license string) error { } } - cksum := sha256.Sum256(buf) + cksumSHA1 := sha1.Sum(buf) //nolint:gosec // used only to produce the sha1 checksum field + cksumSHA256 := sha256.Sum256(buf) log.Info().Str("name", hdr.Name). Int("size", bufsz). - Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksum[:]))). + Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksumSHA256[:]))). Msg("file entry detected") sfile := &spdx.File{ Entity: spdx.Entity{ - Name: hdr.Name[1:], - Checksum: map[string]string{"SHA256": hex.EncodeToString(cksum[:])}, + Name: hdr.Name[1:], + Checksum: map[string]string{ + "SHA1": hex.EncodeToString(cksumSHA1[:]), + "SHA256": hex.EncodeToString(cksumSHA256[:]), + }, }, + LicenseInfoInFile: license, } if err := spkg.AddFile(sfile); err != nil { log.Error().Err(err).Msg("unable to add file to package") @@ -277,6 +284,7 @@ func InstalledPackage(doc *spdx.Document, pkg Package, path string) error { }{ Person: pkg.Maintainer, }, + FilesAnalyzed: true, LicenseDeclared: "unknown", } @@ -309,18 +317,35 @@ func InstalledPackage(doc *spdx.Document, pkg Package, path string) error { } defer fhandle.Close() - shaWriter := sha256.New() - if _, err := io.Copy(shaWriter, fhandle); err != nil { - return err + buf := make([]byte, info.Size()) + + var bufsz int + + if bufsz, err = fhandle.Read(buf); err != nil { + if !errors.Is(err, io.EOF) { + log.Error().Err(err).Str("name", info.Name()).Msg("unable to read content") + + return err + } } - cksum := shaWriter.Sum(nil) + cksumSHA1 := sha1.Sum(buf) //nolint:gosec // used only to produce the sha1 checksum field + cksumSHA256 := sha256.Sum256(buf) + + log.Info().Str("name", info.Name()). + Int("size", bufsz). + Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksumSHA256[:]))). + Msg("file entry detected") sfile := spdx.NewFile() + sfile.LicenseInfoInFile = "unknown" sfile.SetEntity( &spdx.Entity{ - Name: line, - Checksum: map[string]string{"SHA256": hex.EncodeToString(cksum)}, + Name: line, + Checksum: map[string]string{ + "SHA1": hex.EncodeToString(cksumSHA1[:]), + "SHA256": hex.EncodeToString(cksumSHA256[:]), + }, }, ) diff --git a/pkg/distro/rpm/rpm.go b/pkg/distro/rpm/rpm.go index d4e68ae..39ac247 100644 --- a/pkg/distro/rpm/rpm.go +++ b/pkg/distro/rpm/rpm.go @@ -1,8 +1,10 @@ package rpm import ( + "crypto/sha1" //nolint:gosec // used only to produce the sha1 checksum field "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" @@ -87,6 +89,7 @@ func ParsePackage(input, output, author, organization, license string) error { }{ Organization: vendor[0], }, + FilesAnalyzed: true, LicenseDeclared: pkglicense, } @@ -119,18 +122,35 @@ func ParsePackage(input, output, author, organization, license string) error { } defer fhandle.Close() - shaWriter := sha256.New() - if _, err := io.Copy(shaWriter, fhandle); err != nil { - return err + buf := make([]byte, info.Size()) + + var bufsz int + + if bufsz, err = fhandle.Read(buf); err != nil { + if !errors.Is(err, io.EOF) { + log.Error().Err(err).Str("name", finfo.Name()).Msg("unable to read content") + + return err + } } - cksum := shaWriter.Sum(nil) + cksumSHA1 := sha1.Sum(buf) //nolint:gosec // used only to produce the sha1 checksum field + cksumSHA256 := sha256.Sum256(buf) + + log.Info().Str("name", info.Name()). + Int("size", bufsz). + Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksumSHA256[:]))). + Msg("file entry detected") sfile := spdx.NewFile() + sfile.LicenseInfoInFile = pkglicense sfile.SetEntity( &spdx.Entity{ - Name: finfo.Name(), - Checksum: map[string]string{"SHA256": hex.EncodeToString(cksum)}, + Name: finfo.Name(), + Checksum: map[string]string{ + "SHA1": hex.EncodeToString(cksumSHA1[:]), + "SHA256": hex.EncodeToString(cksumSHA256[:]), + }, }, ) @@ -162,6 +182,7 @@ func InstalledPackage(doc *spdx.Document, pkg *rpmdb.PackageInfo) error { }{ Person: pkg.Vendor, }, + FilesAnalyzed: true, LicenseDeclared: pkg.License, } @@ -190,18 +211,35 @@ func InstalledPackage(doc *spdx.Document, pkg *rpmdb.PackageInfo) error { } defer fhandle.Close() - shaWriter := sha256.New() - if _, err := io.Copy(shaWriter, fhandle); err != nil { - return err + buf := make([]byte, info.Size()) + + var bufsz int + + if bufsz, err = fhandle.Read(buf); err != nil { + if !errors.Is(err, io.EOF) { + log.Error().Err(err).Str("name", info.Name()).Msg("unable to read content") + + return err + } } - cksum := shaWriter.Sum(nil) + cksumSHA1 := sha1.Sum(buf) //nolint:gosec // used only to produce the sha1 checksum field + cksumSHA256 := sha256.Sum256(buf) + + log.Info().Str("name", info.Name()). + Int("size", bufsz). + Str("cksum", fmt.Sprintf("SHA256:%s", hex.EncodeToString(cksumSHA256[:]))). + Msg("file entry detected") sfile := spdx.NewFile() + sfile.LicenseInfoInFile = "unknown" sfile.SetEntity( &spdx.Entity{ - Name: ifile.Path, - Checksum: map[string]string{"SHA256": hex.EncodeToString(cksum)}, + Name: ifile.Path, + Checksum: map[string]string{ + "SHA1": hex.EncodeToString(cksumSHA1[:]), + "SHA256": hex.EncodeToString(cksumSHA256[:]), + }, }, ) diff --git a/test/bom.bats b/test/bom.bats index bdce764..a7d0b7e 100644 --- a/test/bom.bats +++ b/test/bom.bats @@ -18,7 +18,7 @@ function teardown() { docker run -v ${TOPDIR}/bin:/opt/bin -v ${BOMD}:/stacker-artifacts -i ubuntu:latest /opt/bin/stacker-bom-linux-amd64 discover -o /stacker-artifacts/discover.json [ -f ${BOMD}/discover.json ] # verify against inventory - ${TOPDIR}/bin/stacker-bom-linux-amd64 verify -i ${BOMD}/discover.json -t ${BOMD}/inventory.json -m ${BOMD}/missing.json + docker run -v ${TOPDIR}/bin:/opt/bin -v ${BOMD}:/stacker-artifacts -i ubuntu:latest /opt/bin/stacker-bom-linux-amd64 verify -i /stacker-artifacts/discover.json -t /stacker-artifacts/inventory.json -m /stacker-artifacts/missing.json [ ! -f ${BOMD}/missing.json ] # push the image skopeo copy --format=oci --dest-tls-verify=false docker://ubuntu:latest docker://${ZOT_HOST}:${ZOT_PORT}/ubuntu:latest