From 7cd3e2b8c8ac6a2cf682bf2551d88d9dfa42503b Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Wed, 7 Aug 2024 17:00:09 -0400 Subject: [PATCH 01/11] feat: add distro mapping for azure linux 3 Signed-off-by: Will Murphy --- grype/db/v5/namespace/index.go | 5 +++++ grype/db/v5/namespace/index_test.go | 16 ++++++++++++++++ grype/distro/distro_test.go | 10 ++++++++++ grype/distro/type.go | 2 ++ 4 files changed, 33 insertions(+) diff --git a/grype/db/v5/namespace/index.go b/grype/db/v5/namespace/index.go index ad6f752e817..454272ee4e2 100644 --- a/grype/db/v5/namespace/index.go +++ b/grype/db/v5/namespace/index.go @@ -134,6 +134,11 @@ func (i *Index) NamespacesForDistro(d *grypeDistro.Distro) []*distro.Namespace { if v, ok := i.byDistroKey[distroKey]; ok { return v } + case grypeDistro.Azure, grypeDistro.Mariner: // mariner was pre-release name for azure + distroKey = fmt.Sprintf("%s:%s", strings.ToLower(string(grypeDistro.Mariner)), d.FullVersion()) + if v, ok := i.byDistroKey[distroKey]; ok { + return v + } } } diff --git a/grype/db/v5/namespace/index_test.go b/grype/db/v5/namespace/index_test.go index 00ac10da3e1..979cd8acc8f 100644 --- a/grype/db/v5/namespace/index_test.go +++ b/grype/db/v5/namespace/index_test.go @@ -137,6 +137,8 @@ func TestIndex_NamespacesForDistro(t *testing.T) { "other-provider:distro:debian:8", "other-provider:distro:redhat:9", "suse:distro:sles:12.5", + "mariner:distro:mariner:2.0", + "mariner:distro:mariner:3.0", "msrc:distro:windows:471816", "ubuntu:distro:ubuntu:18.04", "oracle:distro:oraclelinux:8", @@ -295,6 +297,20 @@ func TestIndex_NamespacesForDistro(t *testing.T) { distro: newDistro(t, osDistro.Mariner, "20.1", []string{}), namespaces: nil, }, + { + name: "Mariner 2.0 matches mariner namespace", + distro: newDistro(t, osDistro.Mariner, "2.0", []string{}), + namespaces: []*distro.Namespace{ + distro.NewNamespace("mariner", "mariner", "2.0"), + }, + }, + { + name: "azurelinux 3 is matched by mariner 3 namespace", + distro: newDistro(t, osDistro.Azure, "3.0", []string{}), + namespaces: []*distro.Namespace{ + distro.NewNamespace("mariner", "mariner", "3.0"), + }, + }, { name: "Oracle Linux Major semvar matches oracle namespace with exact version", distro: newDistro(t, osDistro.OracleLinux, "8", []string{}), diff --git a/grype/distro/distro_test.go b/grype/distro/distro_test.go index 29ac4d94afa..cbeec3af185 100644 --- a/grype/distro/distro_test.go +++ b/grype/distro/distro_test.go @@ -82,6 +82,16 @@ func Test_NewDistroFromRelease(t *testing.T) { expectedRawVersion: "unstable", expectedVersion: "", }, + { + name: "azure linux 3", + release: linux.Release{ + ID: "azurelinux", + Version: "3.0.20240417", + VersionID: "3.0", + }, + expectedType: Azure, + expectedRawVersion: "3.0", + }, } for _, test := range tests { diff --git a/grype/distro/type.go b/grype/distro/type.go index 587bc3168e3..80a8b22b6bc 100644 --- a/grype/distro/type.go +++ b/grype/distro/type.go @@ -25,6 +25,7 @@ const ( Photon Type = "photon" Windows Type = "windows" Mariner Type = "mariner" + Azure Type = "azurelinux" RockyLinux Type = "rockylinux" AlmaLinux Type = "almalinux" Gentoo Type = "gentoo" @@ -74,6 +75,7 @@ var IDMapping = map[string]Type{ "photon": Photon, "windows": Windows, "mariner": Mariner, + "azurelinux": Azure, "rocky": RockyLinux, "almalinux": AlmaLinux, "gentoo": Gentoo, From 352b688ba565a983417e535563dbc63225d6d660 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Wed, 7 Aug 2024 17:00:12 -0400 Subject: [PATCH 02/11] map mariner to azure on write path, not read path Signed-off-by: Will Murphy --- grype/db/v5/namespace/index.go | 5 ----- grype/db/v5/namespace/index_test.go | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/grype/db/v5/namespace/index.go b/grype/db/v5/namespace/index.go index 454272ee4e2..ad6f752e817 100644 --- a/grype/db/v5/namespace/index.go +++ b/grype/db/v5/namespace/index.go @@ -134,11 +134,6 @@ func (i *Index) NamespacesForDistro(d *grypeDistro.Distro) []*distro.Namespace { if v, ok := i.byDistroKey[distroKey]; ok { return v } - case grypeDistro.Azure, grypeDistro.Mariner: // mariner was pre-release name for azure - distroKey = fmt.Sprintf("%s:%s", strings.ToLower(string(grypeDistro.Mariner)), d.FullVersion()) - if v, ok := i.byDistroKey[distroKey]; ok { - return v - } } } diff --git a/grype/db/v5/namespace/index_test.go b/grype/db/v5/namespace/index_test.go index 979cd8acc8f..30b86af8340 100644 --- a/grype/db/v5/namespace/index_test.go +++ b/grype/db/v5/namespace/index_test.go @@ -138,7 +138,7 @@ func TestIndex_NamespacesForDistro(t *testing.T) { "other-provider:distro:redhat:9", "suse:distro:sles:12.5", "mariner:distro:mariner:2.0", - "mariner:distro:mariner:3.0", + "mariner:distro:azurelinux:3.0", "msrc:distro:windows:471816", "ubuntu:distro:ubuntu:18.04", "oracle:distro:oraclelinux:8", @@ -308,7 +308,7 @@ func TestIndex_NamespacesForDistro(t *testing.T) { name: "azurelinux 3 is matched by mariner 3 namespace", distro: newDistro(t, osDistro.Azure, "3.0", []string{}), namespaces: []*distro.Namespace{ - distro.NewNamespace("mariner", "mariner", "3.0"), + distro.NewNamespace("mariner", osDistro.Azure, "3.0"), }, }, { From ede9426be6ee2f9e2e2b0d1004aa609f687182b3 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Wed, 7 Aug 2024 17:00:15 -0400 Subject: [PATCH 03/11] add azure to list of all types Signed-off-by: Will Murphy --- grype/distro/type.go | 1 + 1 file changed, 1 insertion(+) diff --git a/grype/distro/type.go b/grype/distro/type.go index 80a8b22b6bc..69c73c98322 100644 --- a/grype/distro/type.go +++ b/grype/distro/type.go @@ -50,6 +50,7 @@ var All = []Type{ Photon, Windows, Mariner, + Azure, RockyLinux, AlmaLinux, Gentoo, From cb50fc4ce8f68ddcc33c8e2efb1cdbec09e2d8d2 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Wed, 7 Aug 2024 17:00:17 -0400 Subject: [PATCH 04/11] fix unit tests Signed-off-by: Will Murphy --- grype/db/v3/namespace_test.go | 3 ++- grype/distro/distro_test.go | 5 +++++ grype/distro/test-fixtures/os/azurelinux/etc/os-release | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 grype/distro/test-fixtures/os/azurelinux/etc/os-release diff --git a/grype/db/v3/namespace_test.go b/grype/db/v3/namespace_test.go index e152b6750d8..4225fd7841c 100644 --- a/grype/db/v3/namespace_test.go +++ b/grype/db/v3/namespace_test.go @@ -199,8 +199,9 @@ func Test_NamespaceForDistro(t *testing.T) { allDistros.Add(d.String()) } - // TODO: what do we do with mariner + // v3 and older schemas don't include these newer distros: allDistros.Remove(distro.Mariner.String()) + allDistros.Remove(distro.Azure.String()) for _, test := range tests { name := fmt.Sprintf("%s:%s", test.dist, test.version) diff --git a/grype/distro/distro_test.go b/grype/distro/distro_test.go index cbeec3af185..c18be758fa1 100644 --- a/grype/distro/distro_test.go +++ b/grype/distro/distro_test.go @@ -216,6 +216,11 @@ func Test_NewDistroFromRelease_Coverage(t *testing.T) { Type: Mariner, Version: "1.0.0", }, + { + fixture: "test-fixtures/os/azurelinux", + Type: Azure, + Version: "3.0.0", + }, { fixture: "test-fixtures/os/rockylinux", Type: RockyLinux, diff --git a/grype/distro/test-fixtures/os/azurelinux/etc/os-release b/grype/distro/test-fixtures/os/azurelinux/etc/os-release new file mode 100644 index 00000000000..09028dfe941 --- /dev/null +++ b/grype/distro/test-fixtures/os/azurelinux/etc/os-release @@ -0,0 +1,9 @@ +NAME="Microsoft Azure Linux" +VERSION="3.0.20240417" +ID=azurelinux +VERSION_ID="3.0" +PRETTY_NAME="Microsoft Azure Linux 3.0" +ANSI_COLOR="1;34" +HOME_URL="https://aka.ms/azurelinux" +BUG_REPORT_URL="https://aka.ms/azurelinux" +SUPPORT_URL="https://aka.ms/azurelinux" From 17e9964b1480caee566429a133e5b928952d3732 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Thu, 8 Aug 2024 10:48:02 -0400 Subject: [PATCH 05/11] chore: fix line endings in Azure Linux 3.0 example release file Signed-off-by: Will Murphy --- .../test-fixtures/os/azurelinux/etc/os-release | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/grype/distro/test-fixtures/os/azurelinux/etc/os-release b/grype/distro/test-fixtures/os/azurelinux/etc/os-release index 09028dfe941..b7352fb601a 100644 --- a/grype/distro/test-fixtures/os/azurelinux/etc/os-release +++ b/grype/distro/test-fixtures/os/azurelinux/etc/os-release @@ -1,9 +1,9 @@ -NAME="Microsoft Azure Linux" -VERSION="3.0.20240417" -ID=azurelinux -VERSION_ID="3.0" -PRETTY_NAME="Microsoft Azure Linux 3.0" -ANSI_COLOR="1;34" -HOME_URL="https://aka.ms/azurelinux" -BUG_REPORT_URL="https://aka.ms/azurelinux" -SUPPORT_URL="https://aka.ms/azurelinux" +NAME="Microsoft Azure Linux" +VERSION="3.0.20240417" +ID=azurelinux +VERSION_ID="3.0" +PRETTY_NAME="Microsoft Azure Linux 3.0" +ANSI_COLOR="1;34" +HOME_URL="https://aka.ms/azurelinux" +BUG_REPORT_URL="https://aka.ms/azurelinux" +SUPPORT_URL="https://aka.ms/azurelinux" From 2c2062eb947d5a9083785389dc79e56c416eea00 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 24 Sep 2024 07:45:50 -0400 Subject: [PATCH 06/11] chore: update vuln match labels Signed-off-by: Will Murphy --- test/quality/vulnerability-match-labels | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/quality/vulnerability-match-labels b/test/quality/vulnerability-match-labels index 8ad561f7eee..30c7404cd3c 160000 --- a/test/quality/vulnerability-match-labels +++ b/test/quality/vulnerability-match-labels @@ -1 +1 @@ -Subproject commit 8ad561f7eee84ebf3026812dd6f945946a1faa31 +Subproject commit 30c7404cd3c6157db672b5f4a0dde483ddbed52d From 582f9a40df5fb1eea1f50cea9bdd94d303b47c4c Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 24 Sep 2024 07:49:09 -0400 Subject: [PATCH 07/11] chore: add new result set for azure linux 3 Signed-off-by: Will Murphy --- test/quality/.yardstick.yaml | 40 ++++++++++++++++++++++++++++++++++++ test/quality/Makefile | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/test/quality/.yardstick.yaml b/test/quality/.yardstick.yaml index effef38be83..2035d8d04e0 100644 --- a/test/quality/.yardstick.yaml +++ b/test/quality/.yardstick.yaml @@ -128,3 +128,43 @@ result-sets: version: latest+import-db=db.tar.gz takes: SBOM label: reference + pr_vs_latest_via_sbom_2022: + description: "same as 'pr_vs_latest_via_sbom', but includes vulnerabilities from 2022 and before, instead of 2021 and before" + max_year: 2022 + validations: + - max-f1-regression: 0.1 # allowed to regress 0.1 on f1 score + max-new-false-negatives: 10 + max-unlabeled-percent: 0 + max_year: 2022 + matrix: + images: + - docker.io/anchore/test_images:azurelinux3-63671fe@sha256:2d761ba36575ddd4e07d446f4f2a05448298c20e5bdcd3dedfbbc00f9865240d + + tools: + - name: syft + # note: we want to use a fixed version of syft for capturing all results (NOT "latest") + version: v0.98.0 + produces: SBOM + refresh: false + + - name: grype + # note: we import a static (pinned) DB as to prevent changes in the DB from affecting the results. The + # point of this test is to ensure the correctness of the logic in grype itself with real production data. + # By pinning the DB the grype code itself becomes the independent variable under test (and not the + # every-changing DB). That being said, we should be updating this DB periodically to ensure what we + # are testing with is not too stale. + # version: git:current-commit+import-db=db.tar.gz + # for local build of grype, use for example: + version: path:../../+import-db=db.tar.gz + takes: SBOM + label: candidate # is candidate better than the current baseline? + + - name: grype + # note: we import a static (pinned) DB as to prevent changes in the DB from affecting the results. The + # point of this test is to ensure the correctness of the logic in grype itself with real production data. + # By pinning the DB the grype code itself becomes the independent variable under test (and not the + # every-changing DB). That being said, we should be updating this DB periodically to ensure what we + # are testing with is not too stale. + version: latest+import-db=db.tar.gz + takes: SBOM + label: reference # this run is the current baseline diff --git a/test/quality/Makefile b/test/quality/Makefile index 574e579f2d4..782462df9e0 100644 --- a/test/quality/Makefile +++ b/test/quality/Makefile @@ -27,7 +27,7 @@ all: capture validate ## Fetch or capture all data and run all quality checks .PHONY: validate validate: venv $(VULNERABILITY_LABELS)/Makefile ## Run all quality checks against already collected data - $(ACTIVATE_VENV) yardstick validate -r $(RESULT_SET) + $(YARDSTICK) -r $(RESULT_SET) -r $(RESULT_SET)_2022 .PHONY: capture capture: sboms vulns ## Collect and store all syft and grype results From b710e11f39fa9fb4c4549f63b545136c05201a24 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 24 Sep 2024 07:56:40 -0400 Subject: [PATCH 08/11] add and wire up 2022 test set Signed-off-by: Will Murphy --- test/quality/Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/quality/Makefile b/test/quality/Makefile index 782462df9e0..f6fc567673f 100644 --- a/test/quality/Makefile +++ b/test/quality/Makefile @@ -32,21 +32,22 @@ validate: venv $(VULNERABILITY_LABELS)/Makefile ## Run all quality checks agains .PHONY: capture capture: sboms vulns ## Collect and store all syft and grype results -.PHONY: capture +.PHONY: vulns vulns: venv $(TEST_DB) ## Collect and store all grype results $(YARDSTICK) -v result capture -r $(RESULT_SET) + $(YARDSTICK) -v result capture -r $(RESULT_SET)_2022 $(TEST_DB): @curl -o $(TEST_DB) -SsL $(TEST_DB_URL) .PHONY: sboms sboms: $(YARDSTICK_RESULT_DIR) venv clear-results ## Collect and store all syft results (deletes all existing results) - bash -c "make download-sboms || ($(YARDSTICK) -v result capture -r $(RESULT_SET) --only-producers)" + bash -c "make download-sboms || ($(YARDSTICK) -v result capture -r $(RESULT_SET) --only-producers && $(YARDSTICK) -v result capture -r $(RESULT_SET)_2022 --only-producers)" .PHONY: download-sboms download-sboms: $(VULNERABILITY_LABELS)/Makefile cd vulnerability-match-labels && make venv - bash -c "export ORAS_CACHE=$(shell pwd)/.oras-cache && make venv && . vulnerability-match-labels/venv/bin/activate && ./vulnerability-match-labels/sboms.py download -r $(RESULT_SET)" + bash -c "export ORAS_CACHE=$(shell pwd)/.oras-cache && make venv && . vulnerability-match-labels/venv/bin/activate && ./vulnerability-match-labels/sboms.py download -r $(RESULT_SET) && ./vulnerability-match-labels/sboms.py download -r $(RESULT_SET)_2022" venv: venv/touchfile From 51242f5f5eb95f8db1aad5ddca88609cdb7b148a Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 24 Sep 2024 08:20:59 -0400 Subject: [PATCH 09/11] chore: actually validate Signed-off-by: Will Murphy --- test/quality/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/quality/Makefile b/test/quality/Makefile index f6fc567673f..7f5249f4f9e 100644 --- a/test/quality/Makefile +++ b/test/quality/Makefile @@ -27,7 +27,7 @@ all: capture validate ## Fetch or capture all data and run all quality checks .PHONY: validate validate: venv $(VULNERABILITY_LABELS)/Makefile ## Run all quality checks against already collected data - $(YARDSTICK) -r $(RESULT_SET) -r $(RESULT_SET)_2022 + $(YARDSTICK) validate -r $(RESULT_SET) -r $(RESULT_SET)_2022 .PHONY: capture capture: sboms vulns ## Collect and store all syft and grype results From 3ab0e5fd4164e120c3c7a9fce260f87fdc633f97 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 24 Sep 2024 09:38:01 -0400 Subject: [PATCH 10/11] chore: turn off failure on empty match set Right now, the grype PR runner doesn't have a vuln db with Azure Linux 3.0 in it, so this setting needs to be off until the release. Signed-off-by: Will Murphy --- test/quality/.yardstick.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/quality/.yardstick.yaml b/test/quality/.yardstick.yaml index 2035d8d04e0..2a827f89e7c 100644 --- a/test/quality/.yardstick.yaml +++ b/test/quality/.yardstick.yaml @@ -136,6 +136,7 @@ result-sets: max-new-false-negatives: 10 max-unlabeled-percent: 0 max_year: 2022 + fail_on_empty_match_set: false matrix: images: - docker.io/anchore/test_images:azurelinux3-63671fe@sha256:2d761ba36575ddd4e07d446f4f2a05448298c20e5bdcd3dedfbbc00f9865240d From 72aa7e4694b7891c870c4e3b687bea01955d38f2 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 24 Sep 2024 10:26:07 -0400 Subject: [PATCH 11/11] chore: use vuln match labels from main Signed-off-by: Will Murphy --- test/quality/vulnerability-match-labels | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/quality/vulnerability-match-labels b/test/quality/vulnerability-match-labels index 30c7404cd3c..a9a1e820e22 160000 --- a/test/quality/vulnerability-match-labels +++ b/test/quality/vulnerability-match-labels @@ -1 +1 @@ -Subproject commit 30c7404cd3c6157db672b5f4a0dde483ddbed52d +Subproject commit a9a1e820e22d52c94bd70dd5bfce8f29bbdb7ce4