From f2a12f5f90f940effe5dfc763025a70b127ec539 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Fri, 20 Oct 2023 10:43:15 +0900 Subject: [PATCH] refactor: define a new struct for scan targets (#5397) --- pkg/fanal/types/artifact.go | 2 +- pkg/scanner/langpkg/scan.go | 35 ++++---- pkg/scanner/local/scan.go | 174 +++++++++++++++++++++++------------- pkg/scanner/ospkg/scan.go | 58 ++++-------- pkg/types/scan.go | 33 +++++++ pkg/types/scanoptions.go | 17 ---- 6 files changed, 179 insertions(+), 140 deletions(-) create mode 100644 pkg/types/scan.go delete mode 100644 pkg/types/scanoptions.go diff --git a/pkg/fanal/types/artifact.go b/pkg/fanal/types/artifact.go index c60d68fc75e6..7db2cdbf1949 100644 --- a/pkg/fanal/types/artifact.go +++ b/pkg/fanal/types/artifact.go @@ -264,7 +264,7 @@ type BlobInfo struct { CustomResources []CustomResource `json:",omitempty"` } -// ArtifactDetail is generated by applying blobs +// ArtifactDetail represents the analysis result. type ArtifactDetail struct { OS OS `json:",omitempty"` Repository *Repository `json:",omitempty"` diff --git a/pkg/scanner/langpkg/scan.go b/pkg/scanner/langpkg/scan.go index df16c0c78476..9480c8a13614 100644 --- a/pkg/scanner/langpkg/scan.go +++ b/pkg/scanner/langpkg/scan.go @@ -23,8 +23,8 @@ var ( ) type Scanner interface { - Packages(detail ftypes.ArtifactDetail, options types.ScanOptions) types.Results - Scan(detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Results, error) + Packages(target types.ScanTarget, options types.ScanOptions) types.Results + Scan(target types.ScanTarget, options types.ScanOptions) (types.Results, error) } type scanner struct{} @@ -33,20 +33,15 @@ func NewScanner() Scanner { return &scanner{} } -func (s *scanner) Packages(detail ftypes.ArtifactDetail, _ types.ScanOptions) types.Results { +func (s *scanner) Packages(target types.ScanTarget, _ types.ScanOptions) types.Results { var results types.Results - for _, app := range detail.Applications { + for _, app := range target.Applications { if len(app.Libraries) == 0 { continue } - target := app.FilePath - if t, ok := PkgTargets[app.Type]; ok && target == "" { - // When the file path is empty, we will overwrite it with the pre-defined value. - target = t - } results = append(results, types.Result{ - Target: target, + Target: targetName(app.Type, app.FilePath), Class: types.ClassLangPkg, Type: app.Type, Packages: app.Libraries, @@ -55,8 +50,8 @@ func (s *scanner) Packages(detail ftypes.ArtifactDetail, _ types.ScanOptions) ty return results } -func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types.Results, error) { - apps := detail.Applications +func (s *scanner) Scan(target types.ScanTarget, _ types.ScanOptions) (types.Results, error) { + apps := target.Applications log.Logger.Infof("Number of language-specific files: %d", len(apps)) if len(apps) == 0 { return nil, nil @@ -83,14 +78,8 @@ func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types continue } - target := app.FilePath - if t, ok := PkgTargets[app.Type]; ok && target == "" { - // When the file path is empty, we will overwrite it with the pre-defined value. - target = t - } - results = append(results, types.Result{ - Target: target, + Target: targetName(app.Type, app.FilePath), Vulnerabilities: vulns, Class: types.ClassLangPkg, Type: app.Type, @@ -101,3 +90,11 @@ func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types }) return results, nil } + +func targetName(appType ftypes.LangType, filePath string) string { + if t, ok := PkgTargets[appType]; ok && filePath == "" { + // When the file path is empty, we will overwrite it with the pre-defined value. + return t + } + return filePath +} diff --git a/pkg/scanner/local/scan.go b/pkg/scanner/local/scan.go index 5539bf23efc0..63822852cb71 100644 --- a/pkg/scanner/local/scan.go +++ b/pkg/scanner/local/scan.go @@ -57,24 +57,25 @@ func NewScanner(a applier.Applier, osPkgScanner ospkg.Scanner, langPkgScanner la } // Scan scans the artifact and return results. -func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys []string, options types.ScanOptions) (types.Results, ftypes.OS, error) { - artifactDetail, err := s.applier.ApplyLayers(artifactKey, blobKeys) +func (s Scanner) Scan(ctx context.Context, targetName, artifactKey string, blobKeys []string, options types.ScanOptions) ( + types.Results, ftypes.OS, error) { + detail, err := s.applier.ApplyLayers(artifactKey, blobKeys) switch { case errors.Is(err, analyzer.ErrUnknownOS): log.Logger.Debug("OS is not detected.") // Packages may contain OS-independent binary information even though OS is not detected. - if len(artifactDetail.Packages) != 0 { - artifactDetail.OS = ftypes.OS{Family: "none"} + if len(detail.Packages) != 0 { + detail.OS = ftypes.OS{Family: "none"} } // If OS is not detected and repositories are detected, we'll try to use repositories as OS. - if artifactDetail.Repository != nil { - log.Logger.Debugf("Package repository: %s %s", artifactDetail.Repository.Family, artifactDetail.Repository.Release) - log.Logger.Debugf("Assuming OS is %s %s.", artifactDetail.Repository.Family, artifactDetail.Repository.Release) - artifactDetail.OS = ftypes.OS{ - Family: artifactDetail.Repository.Family, - Name: artifactDetail.Repository.Release, + if detail.Repository != nil { + log.Logger.Debugf("Package repository: %s %s", detail.Repository.Family, detail.Repository.Release) + log.Logger.Debugf("Assuming OS is %s %s.", detail.Repository.Family, detail.Repository.Release) + detail.OS = ftypes.OS{ + Family: detail.Repository.Family, + Name: detail.Repository.Release, } } case errors.Is(err, analyzer.ErrNoPkgsDetected): @@ -84,29 +85,46 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys return nil, ftypes.OS{}, xerrors.Errorf("failed to apply layers: %w", err) } + target := types.ScanTarget{ + Name: targetName, + OS: detail.OS, + Repository: detail.Repository, + Packages: mergePkgs(detail.Packages, detail.ImageConfig.Packages, options), + Applications: detail.Applications, + Misconfigurations: mergeMisconfigurations(targetName, detail), + Secrets: mergeSecrets(targetName, detail), + Licenses: detail.Licenses, + CustomResources: detail.CustomResources, + } + + return s.ScanTarget(ctx, target, options) +} + +func (s Scanner) ScanTarget(ctx context.Context, target types.ScanTarget, options types.ScanOptions) (types.Results, ftypes.OS, error) { var eosl bool var results, pkgResults types.Results + var err error // By default, we need to remove dev dependencies from the result // IncludeDevDeps option allows you not to remove them - excludeDevDeps(artifactDetail.Applications, options.IncludeDevDeps) + excludeDevDeps(target.Applications, options.IncludeDevDeps) // Fill OS packages and language-specific packages if options.ListAllPackages { - if res := s.osPkgScanner.Packages(target, artifactDetail, options); len(res.Packages) != 0 { + if res := s.osPkgScanner.Packages(target, options); len(res.Packages) != 0 { pkgResults = append(pkgResults, res) } - pkgResults = append(pkgResults, s.langPkgScanner.Packages(artifactDetail, options)...) + pkgResults = append(pkgResults, s.langPkgScanner.Packages(target, options)...) } // Scan packages for vulnerabilities if options.Scanners.Enabled(types.VulnerabilityScanner) { var vulnResults types.Results - vulnResults, eosl, err = s.scanVulnerabilities(target, artifactDetail, options) + vulnResults, eosl, err = s.scanVulnerabilities(target, options) if err != nil { return nil, ftypes.OS{}, xerrors.Errorf("failed to detect vulnerabilities: %w", err) } - artifactDetail.OS.Eosl = eosl + target.OS.Eosl = eosl // Merge package results into vulnerability results mergedResults := s.fillPkgsInVulns(pkgResults, vulnResults) @@ -117,45 +135,20 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys results = append(results, pkgResults...) } - // Scan IaC config files - if ShouldScanMisconfigOrRbac(options.Scanners) { - configResults := s.MisconfsToResults(artifactDetail.Misconfigurations) - results = append(results, configResults...) - } + // Store misconfigurations + results = append(results, s.misconfsToResults(target.Misconfigurations, options)...) - // Scan secrets - if options.Scanners.Enabled(types.SecretScanner) { - secretResults := s.secretsToResults(artifactDetail.Secrets) - results = append(results, secretResults...) - } + // Store secrets + results = append(results, s.secretsToResults(target.Secrets, options)...) // Scan licenses - if options.Scanners.Enabled(types.LicenseScanner) { - licenseResults := s.scanLicenses(artifactDetail, options.LicenseCategories) - results = append(results, licenseResults...) - } - - // Scan misconfigurations on container image config - if options.ImageConfigScanners.Enabled(types.MisconfigScanner) { - if im := artifactDetail.ImageConfig.Misconfiguration; im != nil { - im.FilePath = target // Set the target name to the file path as container image config is not a real file. - results = append(results, s.MisconfsToResults([]ftypes.Misconfiguration{*im})...) - } - } - - // Scan secrets on container image config - if options.ImageConfigScanners.Enabled(types.SecretScanner) { - if is := artifactDetail.ImageConfig.Secret; is != nil { - is.FilePath = target // Set the target name to the file path as container image config is not a real file. - results = append(results, s.secretsToResults([]ftypes.Secret{*is})...) - } - } + results = append(results, s.scanLicenses(target, options)...) // For WASM plugins and custom analyzers - if len(artifactDetail.CustomResources) != 0 { + if len(target.CustomResources) != 0 { results = append(results, types.Result{ Class: types.ClassCustom, - CustomResources: artifactDetail.CustomResources, + CustomResources: target.CustomResources, }) } @@ -170,16 +163,16 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys return nil, ftypes.OS{}, xerrors.Errorf("post scan error: %w", err) } - return results, artifactDetail.OS, nil + return results, target.OS, nil } -func (s Scanner) scanVulnerabilities(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) ( +func (s Scanner) scanVulnerabilities(target types.ScanTarget, options types.ScanOptions) ( types.Results, bool, error) { var eosl bool var results types.Results if slices.Contains(options.VulnType, types.VulnTypeOS) { - vuln, detectedEOSL, err := s.osPkgScanner.Scan(target, detail, options) + vuln, detectedEOSL, err := s.osPkgScanner.Scan(target, options) if err != nil { return nil, false, xerrors.Errorf("unable to scan OS packages: %w", err) } else if vuln.Target != "" { @@ -189,7 +182,7 @@ func (s Scanner) scanVulnerabilities(target string, detail ftypes.ArtifactDetail } if slices.Contains(options.VulnType, types.VulnTypeLibrary) { - vulns, err := s.langPkgScanner.Scan(detail, options) + vulns, err := s.langPkgScanner.Scan(target, options) if err != nil { return nil, false, xerrors.Errorf("failed to scan application libraries: %w", err) } @@ -217,6 +210,14 @@ func (s Scanner) fillPkgsInVulns(pkgResults, vulnResults types.Results) types.Re return results } +func (s Scanner) misconfsToResults(misconfs []ftypes.Misconfiguration, options types.ScanOptions) types.Results { + if !ShouldScanMisconfigOrRbac(options.Scanners) { + return nil + } + + return s.MisconfsToResults(misconfs) +} + // MisconfsToResults is exported for trivy-plugin-aqua purposes only func (s Scanner) MisconfsToResults(misconfs []ftypes.Misconfiguration) types.Results { log.Logger.Infof("Detected config files: %d", len(misconfs)) @@ -254,7 +255,11 @@ func (s Scanner) MisconfsToResults(misconfs []ftypes.Misconfiguration) types.Res return results } -func (s Scanner) secretsToResults(secrets []ftypes.Secret) types.Results { +func (s Scanner) secretsToResults(secrets []ftypes.Secret, options types.ScanOptions) types.Results { + if !options.Scanners.Enabled(types.SecretScanner) { + return nil + } + var results types.Results for _, secret := range secrets { log.Logger.Debugf("Secret file: %s", secret.FilePath) @@ -268,15 +273,17 @@ func (s Scanner) secretsToResults(secrets []ftypes.Secret) types.Results { return results } -func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, - categories map[ftypes.LicenseCategory][]string) types.Results { - scanner := licensing.NewScanner(categories) +func (s Scanner) scanLicenses(target types.ScanTarget, options types.ScanOptions) types.Results { + if !options.Scanners.Enabled(types.LicenseScanner) { + return nil + } var results types.Results + scanner := licensing.NewScanner(options.LicenseCategories) // License - OS packages var osPkgLicenses []types.DetectedLicense - for _, pkg := range detail.Packages { + for _, pkg := range target.Packages { for _, license := range pkg.Licenses { category, severity := scanner.Scan(license) osPkgLicenses = append(osPkgLicenses, types.DetectedLicense{ @@ -296,7 +303,7 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, }) // License - language-specific packages - for _, app := range detail.Applications { + for _, app := range target.Applications { var langLicenses []types.DetectedLicense for _, lib := range app.Libraries { for _, license := range lib.Licenses { @@ -311,13 +318,13 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, } } - target := app.FilePath - if t, ok := langpkg.PkgTargets[app.Type]; ok && target == "" { + targetName := app.FilePath + if t, ok := langpkg.PkgTargets[app.Type]; ok && targetName == "" { // When the file path is empty, we will overwrite it with the pre-defined value. - target = t + targetName = t } results = append(results, types.Result{ - Target: target, + Target: targetName, Class: types.ClassLicense, Licenses: langLicenses, }) @@ -325,7 +332,7 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, // License - file header or license file var fileLicenses []types.DetectedLicense - for _, license := range detail.Licenses { + for _, license := range target.Licenses { for _, finding := range license.Findings { category, severity := scanner.Scan(finding.Name) fileLicenses = append(fileLicenses, types.DetectedLicense{ @@ -420,3 +427,46 @@ func excludeDevDeps(apps []ftypes.Application, include bool) { }) } } + +func mergePkgs(pkgs, pkgsFromCommands []ftypes.Package, options types.ScanOptions) []ftypes.Package { + if !options.ScanRemovedPackages || len(pkgsFromCommands) == 0 { + return pkgs + } + + // pkg has priority over pkgsFromCommands + uniqPkgs := make(map[string]struct{}) + for _, pkg := range pkgs { + uniqPkgs[pkg.Name] = struct{}{} + } + for _, pkg := range pkgsFromCommands { + if _, ok := uniqPkgs[pkg.Name]; ok { + continue + } + pkgs = append(pkgs, pkg) + } + return pkgs +} + +// mergeMisconfigurations merges misconfigurations on container image config +func mergeMisconfigurations(targetName string, detail ftypes.ArtifactDetail) []ftypes.Misconfiguration { + if detail.ImageConfig.Misconfiguration == nil { + return detail.Misconfigurations + } + + // Append misconfigurations on container image config + misconf := detail.ImageConfig.Misconfiguration + misconf.FilePath = targetName // Set the target name to the file path as container image config is not a real file. + return append(detail.Misconfigurations, *misconf) +} + +// mergeSecrets merges secrets on container image config. +func mergeSecrets(targetName string, detail ftypes.ArtifactDetail) []ftypes.Secret { + if detail.ImageConfig.Secret == nil { + return detail.Secrets + } + + // Append secrets on container image config + secret := detail.ImageConfig.Secret + secret.FilePath = targetName // Set the target name to the file path as container image config is not a real file. + return append(detail.Secrets, *secret) +} diff --git a/pkg/scanner/ospkg/scan.go b/pkg/scanner/ospkg/scan.go index ed255dd362e3..a07d6e6b2091 100644 --- a/pkg/scanner/ospkg/scan.go +++ b/pkg/scanner/ospkg/scan.go @@ -8,14 +8,13 @@ import ( "golang.org/x/xerrors" ospkgDetector "github.com/aquasecurity/trivy/pkg/detector/ospkg" - ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/types" ) type Scanner interface { - Packages(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) types.Result - Scan(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Result, bool, error) + Packages(target types.ScanTarget, options types.ScanOptions) types.Result + Scan(target types.ScanTarget, options types.ScanOptions) (types.Result, bool, error) } type scanner struct{} @@ -24,68 +23,45 @@ func NewScanner() Scanner { return &scanner{} } -func (s *scanner) Packages(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) types.Result { - if len(detail.Packages) == 0 || !detail.OS.Detected() { +func (s *scanner) Packages(target types.ScanTarget, _ types.ScanOptions) types.Result { + if len(target.Packages) == 0 || !target.OS.Detected() { return types.Result{} } - pkgs := detail.Packages - if options.ScanRemovedPackages { - pkgs = mergePkgs(pkgs, detail.ImageConfig.Packages) - } - sort.Sort(pkgs) + sort.Sort(target.Packages) return types.Result{ - Target: fmt.Sprintf("%s (%s %s)", target, detail.OS.Family, detail.OS.Name), + Target: fmt.Sprintf("%s (%s %s)", target.Name, target.OS.Family, target.OS.Name), Class: types.ClassOSPkg, - Type: detail.OS.Family, - Packages: pkgs, + Type: target.OS.Family, + Packages: target.Packages, } } -func (s *scanner) Scan(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Result, bool, error) { - if !detail.OS.Detected() { +func (s *scanner) Scan(target types.ScanTarget, _ types.ScanOptions) (types.Result, bool, error) { + if !target.OS.Detected() { log.Logger.Debug("Detected OS: unknown") return types.Result{}, false, nil } - log.Logger.Infof("Detected OS: %s", detail.OS.Family) - - pkgs := detail.Packages - if options.ScanRemovedPackages { - pkgs = mergePkgs(pkgs, detail.ImageConfig.Packages) - } + log.Logger.Infof("Detected OS: %s", target.OS.Family) - if detail.OS.Extended { + if target.OS.Extended { // TODO: move the logic to each detector - detail.OS.Name += "-ESM" + target.OS.Name += "-ESM" } - vulns, eosl, err := ospkgDetector.Detect("", detail.OS.Family, detail.OS.Name, detail.Repository, time.Time{}, pkgs) + vulns, eosl, err := ospkgDetector.Detect("", target.OS.Family, target.OS.Name, target.Repository, time.Time{}, + target.Packages) if err == ospkgDetector.ErrUnsupportedOS { return types.Result{}, false, nil } else if err != nil { return types.Result{}, false, xerrors.Errorf("failed vulnerability detection of OS packages: %w", err) } - artifactDetail := fmt.Sprintf("%s (%s %s)", target, detail.OS.Family, detail.OS.Name) + artifactDetail := fmt.Sprintf("%s (%s %s)", target.Name, target.OS.Family, target.OS.Name) return types.Result{ Target: artifactDetail, Vulnerabilities: vulns, Class: types.ClassOSPkg, - Type: detail.OS.Family, + Type: target.OS.Family, }, eosl, nil } - -func mergePkgs(pkgs, pkgsFromCommands []ftypes.Package) []ftypes.Package { - // pkg has priority over pkgsFromCommands - uniqPkgs := make(map[string]struct{}) - for _, pkg := range pkgs { - uniqPkgs[pkg.Name] = struct{}{} - } - for _, pkg := range pkgsFromCommands { - if _, ok := uniqPkgs[pkg.Name]; ok { - continue - } - pkgs = append(pkgs, pkg) - } - return pkgs -} diff --git a/pkg/types/scan.go b/pkg/types/scan.go new file mode 100644 index 000000000000..0b5833b12faa --- /dev/null +++ b/pkg/types/scan.go @@ -0,0 +1,33 @@ +package types + +import ( + "github.com/aquasecurity/trivy/pkg/fanal/types" +) + +// ScanTarget holds the attributes for scanning. +type ScanTarget struct { + Name string // container image name, file path, etc + OS types.OS + Repository *types.Repository + Packages types.Packages + Applications []types.Application + Misconfigurations []types.Misconfiguration + Secrets []types.Secret + Licenses []types.LicenseFile + + // CustomResources hold analysis results from custom analyzers. + // It is for extensibility and not used in OSS. + CustomResources []types.CustomResource +} + +// ScanOptions holds the attributes for scanning vulnerabilities +type ScanOptions struct { + VulnType []string + Scanners Scanners + ImageConfigScanners Scanners // Scanners for container image configuration + ScanRemovedPackages bool + ListAllPackages bool + LicenseCategories map[types.LicenseCategory][]string + FilePatterns []string + IncludeDevDeps bool +} diff --git a/pkg/types/scanoptions.go b/pkg/types/scanoptions.go deleted file mode 100644 index 1e075bba6839..000000000000 --- a/pkg/types/scanoptions.go +++ /dev/null @@ -1,17 +0,0 @@ -package types - -import ( - "github.com/aquasecurity/trivy/pkg/fanal/types" -) - -// ScanOptions holds the attributes for scanning vulnerabilities -type ScanOptions struct { - VulnType []string - Scanners Scanners - ImageConfigScanners Scanners // Scanners for container image configuration - ScanRemovedPackages bool - ListAllPackages bool - LicenseCategories map[types.LicenseCategory][]string - FilePatterns []string - IncludeDevDeps bool -}