diff --git a/grype/db/v3/namespace_test.go b/grype/db/v3/namespace_test.go index e152b6750d8..4225fd7841c 100644 --- a/grype/db/v3/namespace_test.go +++ b/grype/db/v3/namespace_test.go @@ -199,8 +199,9 @@ func Test_NamespaceForDistro(t *testing.T) { allDistros.Add(d.String()) } - // TODO: what do we do with mariner + // v3 and older schemas don't include these newer distros: allDistros.Remove(distro.Mariner.String()) + allDistros.Remove(distro.Azure.String()) for _, test := range tests { name := fmt.Sprintf("%s:%s", test.dist, test.version) diff --git a/grype/db/v5/namespace/index_test.go b/grype/db/v5/namespace/index_test.go index 00ac10da3e1..30b86af8340 100644 --- a/grype/db/v5/namespace/index_test.go +++ b/grype/db/v5/namespace/index_test.go @@ -137,6 +137,8 @@ func TestIndex_NamespacesForDistro(t *testing.T) { "other-provider:distro:debian:8", "other-provider:distro:redhat:9", "suse:distro:sles:12.5", + "mariner:distro:mariner:2.0", + "mariner:distro:azurelinux:3.0", "msrc:distro:windows:471816", "ubuntu:distro:ubuntu:18.04", "oracle:distro:oraclelinux:8", @@ -295,6 +297,20 @@ func TestIndex_NamespacesForDistro(t *testing.T) { distro: newDistro(t, osDistro.Mariner, "20.1", []string{}), namespaces: nil, }, + { + name: "Mariner 2.0 matches mariner namespace", + distro: newDistro(t, osDistro.Mariner, "2.0", []string{}), + namespaces: []*distro.Namespace{ + distro.NewNamespace("mariner", "mariner", "2.0"), + }, + }, + { + name: "azurelinux 3 is matched by mariner 3 namespace", + distro: newDistro(t, osDistro.Azure, "3.0", []string{}), + namespaces: []*distro.Namespace{ + distro.NewNamespace("mariner", osDistro.Azure, "3.0"), + }, + }, { name: "Oracle Linux Major semvar matches oracle namespace with exact version", distro: newDistro(t, osDistro.OracleLinux, "8", []string{}), diff --git a/grype/distro/distro_test.go b/grype/distro/distro_test.go index 29ac4d94afa..c18be758fa1 100644 --- a/grype/distro/distro_test.go +++ b/grype/distro/distro_test.go @@ -82,6 +82,16 @@ func Test_NewDistroFromRelease(t *testing.T) { expectedRawVersion: "unstable", expectedVersion: "", }, + { + name: "azure linux 3", + release: linux.Release{ + ID: "azurelinux", + Version: "3.0.20240417", + VersionID: "3.0", + }, + expectedType: Azure, + expectedRawVersion: "3.0", + }, } for _, test := range tests { @@ -206,6 +216,11 @@ func Test_NewDistroFromRelease_Coverage(t *testing.T) { Type: Mariner, Version: "1.0.0", }, + { + fixture: "test-fixtures/os/azurelinux", + Type: Azure, + Version: "3.0.0", + }, { fixture: "test-fixtures/os/rockylinux", Type: RockyLinux, diff --git a/grype/distro/test-fixtures/os/azurelinux/etc/os-release b/grype/distro/test-fixtures/os/azurelinux/etc/os-release new file mode 100644 index 00000000000..b7352fb601a --- /dev/null +++ b/grype/distro/test-fixtures/os/azurelinux/etc/os-release @@ -0,0 +1,9 @@ +NAME="Microsoft Azure Linux" +VERSION="3.0.20240417" +ID=azurelinux +VERSION_ID="3.0" +PRETTY_NAME="Microsoft Azure Linux 3.0" +ANSI_COLOR="1;34" +HOME_URL="https://aka.ms/azurelinux" +BUG_REPORT_URL="https://aka.ms/azurelinux" +SUPPORT_URL="https://aka.ms/azurelinux" diff --git a/grype/distro/type.go b/grype/distro/type.go index 587bc3168e3..69c73c98322 100644 --- a/grype/distro/type.go +++ b/grype/distro/type.go @@ -25,6 +25,7 @@ const ( Photon Type = "photon" Windows Type = "windows" Mariner Type = "mariner" + Azure Type = "azurelinux" RockyLinux Type = "rockylinux" AlmaLinux Type = "almalinux" Gentoo Type = "gentoo" @@ -49,6 +50,7 @@ var All = []Type{ Photon, Windows, Mariner, + Azure, RockyLinux, AlmaLinux, Gentoo, @@ -74,6 +76,7 @@ var IDMapping = map[string]Type{ "photon": Photon, "windows": Windows, "mariner": Mariner, + "azurelinux": Azure, "rocky": RockyLinux, "almalinux": AlmaLinux, "gentoo": Gentoo, diff --git a/test/quality/.yardstick.yaml b/test/quality/.yardstick.yaml index effef38be83..2a827f89e7c 100644 --- a/test/quality/.yardstick.yaml +++ b/test/quality/.yardstick.yaml @@ -128,3 +128,44 @@ result-sets: version: latest+import-db=db.tar.gz takes: SBOM label: reference + pr_vs_latest_via_sbom_2022: + description: "same as 'pr_vs_latest_via_sbom', but includes vulnerabilities from 2022 and before, instead of 2021 and before" + max_year: 2022 + validations: + - max-f1-regression: 0.1 # allowed to regress 0.1 on f1 score + max-new-false-negatives: 10 + max-unlabeled-percent: 0 + max_year: 2022 + fail_on_empty_match_set: false + matrix: + images: + - docker.io/anchore/test_images:azurelinux3-63671fe@sha256:2d761ba36575ddd4e07d446f4f2a05448298c20e5bdcd3dedfbbc00f9865240d + + tools: + - name: syft + # note: we want to use a fixed version of syft for capturing all results (NOT "latest") + version: v0.98.0 + produces: SBOM + refresh: false + + - name: grype + # note: we import a static (pinned) DB as to prevent changes in the DB from affecting the results. The + # point of this test is to ensure the correctness of the logic in grype itself with real production data. + # By pinning the DB the grype code itself becomes the independent variable under test (and not the + # every-changing DB). That being said, we should be updating this DB periodically to ensure what we + # are testing with is not too stale. + # version: git:current-commit+import-db=db.tar.gz + # for local build of grype, use for example: + version: path:../../+import-db=db.tar.gz + takes: SBOM + label: candidate # is candidate better than the current baseline? + + - name: grype + # note: we import a static (pinned) DB as to prevent changes in the DB from affecting the results. The + # point of this test is to ensure the correctness of the logic in grype itself with real production data. + # By pinning the DB the grype code itself becomes the independent variable under test (and not the + # every-changing DB). That being said, we should be updating this DB periodically to ensure what we + # are testing with is not too stale. + version: latest+import-db=db.tar.gz + takes: SBOM + label: reference # this run is the current baseline diff --git a/test/quality/Makefile b/test/quality/Makefile index 574e579f2d4..7f5249f4f9e 100644 --- a/test/quality/Makefile +++ b/test/quality/Makefile @@ -27,26 +27,27 @@ all: capture validate ## Fetch or capture all data and run all quality checks .PHONY: validate validate: venv $(VULNERABILITY_LABELS)/Makefile ## Run all quality checks against already collected data - $(ACTIVATE_VENV) yardstick validate -r $(RESULT_SET) + $(YARDSTICK) validate -r $(RESULT_SET) -r $(RESULT_SET)_2022 .PHONY: capture capture: sboms vulns ## Collect and store all syft and grype results -.PHONY: capture +.PHONY: vulns vulns: venv $(TEST_DB) ## Collect and store all grype results $(YARDSTICK) -v result capture -r $(RESULT_SET) + $(YARDSTICK) -v result capture -r $(RESULT_SET)_2022 $(TEST_DB): @curl -o $(TEST_DB) -SsL $(TEST_DB_URL) .PHONY: sboms sboms: $(YARDSTICK_RESULT_DIR) venv clear-results ## Collect and store all syft results (deletes all existing results) - bash -c "make download-sboms || ($(YARDSTICK) -v result capture -r $(RESULT_SET) --only-producers)" + bash -c "make download-sboms || ($(YARDSTICK) -v result capture -r $(RESULT_SET) --only-producers && $(YARDSTICK) -v result capture -r $(RESULT_SET)_2022 --only-producers)" .PHONY: download-sboms download-sboms: $(VULNERABILITY_LABELS)/Makefile cd vulnerability-match-labels && make venv - bash -c "export ORAS_CACHE=$(shell pwd)/.oras-cache && make venv && . vulnerability-match-labels/venv/bin/activate && ./vulnerability-match-labels/sboms.py download -r $(RESULT_SET)" + bash -c "export ORAS_CACHE=$(shell pwd)/.oras-cache && make venv && . vulnerability-match-labels/venv/bin/activate && ./vulnerability-match-labels/sboms.py download -r $(RESULT_SET) && ./vulnerability-match-labels/sboms.py download -r $(RESULT_SET)_2022" venv: venv/touchfile diff --git a/test/quality/vulnerability-match-labels b/test/quality/vulnerability-match-labels index 8ad561f7eee..a9a1e820e22 160000 --- a/test/quality/vulnerability-match-labels +++ b/test/quality/vulnerability-match-labels @@ -1 +1 @@ -Subproject commit 8ad561f7eee84ebf3026812dd6f945946a1faa31 +Subproject commit a9a1e820e22d52c94bd70dd5bfce8f29bbdb7ce4