From c14b3427c4a47dc1f7e62f60bcb58705f8768988 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 5 Dec 2024 10:49:24 +0100 Subject: [PATCH] feat: allow processing CDX 1.6 --- Cargo.lock | 68 +++++++++++++- Cargo.toml | 1 + entity/src/sbom.rs | 2 +- migration/src/lib.rs | 2 + .../m0000760_alter_sbom_alter_document_id.rs | 55 ++++++++++++ modules/fundamental/Cargo.toml | 4 +- .../src/advisory/endpoints/test.rs | 14 ++- modules/fundamental/src/ai/service/test.rs | 2 +- .../fundamental/src/product/service/test.rs | 4 +- modules/fundamental/src/sbom/model/mod.rs | 2 +- modules/fundamental/src/sbom/service/sbom.rs | 16 ++-- .../src/vulnerability/service/test.rs | 4 +- .../fundamental/tests/sbom/cyclonedx/mod.rs | 6 +- modules/fundamental/tests/sbom/graph.rs | 28 +++--- modules/fundamental/tests/sbom/mod.rs | 3 +- modules/fundamental/tests/sbom/reingest.rs | 20 ++--- modules/ingestor/Cargo.toml | 1 + modules/ingestor/src/graph/sbom/cyclonedx.rs | 88 ++++++++++--------- modules/ingestor/src/graph/sbom/mod.rs | 4 +- modules/ingestor/src/model.rs | 2 +- .../src/service/advisory/csaf/loader.rs | 2 +- .../src/service/advisory/cve/loader.rs | 2 +- .../src/service/advisory/osv/loader.rs | 2 +- modules/ingestor/src/service/mod.rs | 2 +- .../src/service/sbom/clearly_defined.rs | 2 +- .../service/sbom/clearly_defined_curation.rs | 8 +- .../ingestor/src/service/sbom/cyclonedx.rs | 12 ++- modules/ingestor/src/service/sbom/spdx.rs | 10 ++- modules/ingestor/src/service/weakness/mod.rs | 2 +- openapi.yaml | 10 ++- test-context/src/lib.rs | 2 +- 31 files changed, 267 insertions(+), 113 deletions(-) create mode 100644 migration/src/m0000760_alter_sbom_alter_document_id.rs diff --git a/Cargo.lock b/Cargo.lock index 197cded83..525d2e0a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5375,6 +5375,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn 2.0.90", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -6442,6 +6452,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemafy_core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bec29dddcfe60f92f3c0d422707b8b56473983ef0481df8d5236ed3ab8fdf24" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "schemafy_lib" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3d87f1df246a9b7e2bfd1f4ee5f88e48b11ef9cfc62e63f0dead255b1a6f5f" +dependencies = [ + "Inflector", + "proc-macro2", + "quote", + "schemafy_core", + "serde", + "serde_derive", + "serde_json", + "syn 1.0.109", + "uriparse", +] + [[package]] name = "schemars" version = "0.8.21" @@ -6793,6 +6830,24 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-cyclonedx" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d014c70fa69c686429c29113386afc9fabb8c2be00ee53675ff93a55054934" +dependencies = [ + "anyhow", + "derive_builder 0.20.2", + "prettyplease", + "proc-macro2", + "quote", + "schemafy_lib", + "serde", + "serde_json", + "syn 2.0.90", + "thiserror 1.0.69", +] + [[package]] name = "serde-value" version = "0.7.0" @@ -8446,7 +8501,6 @@ dependencies = [ "criterion", "csaf", "cve", - "cyclonedx-bom", "futures-util", "hex", "humantime", @@ -8464,6 +8518,7 @@ dependencies = [ "sea-query", "semver", "serde", + "serde-cyclonedx", "serde_json", "serde_yml", "sha2", @@ -8598,6 +8653,7 @@ dependencies = [ "sea-query", "semver", "serde", + "serde-cyclonedx", "serde_json", "serde_yml", "spdx", @@ -8929,6 +8985,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "uriparse" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" +dependencies = [ + "fnv", + "lazy_static", +] + [[package]] name = "url" version = "2.5.4" diff --git a/Cargo.toml b/Cargo.toml index 05b0ceeeb..a8b9862b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,6 +115,7 @@ sea-orm-migration = "1" sea-query = "0.32.0" semver = "1" serde = "1.0.183" +serde-cyclonedx = "0.9.1" serde_json = "1.0.114" serde_with = "3.11.0" serde_yml = "0.0.12" diff --git a/entity/src/sbom.rs b/entity/src/sbom.rs index 9f5572db8..fddb3dbb5 100644 --- a/entity/src/sbom.rs +++ b/entity/src/sbom.rs @@ -12,7 +12,7 @@ pub struct Model { pub sbom_id: Uuid, pub node_id: String, - pub document_id: String, + pub document_id: Option, pub published: Option, pub authors: Vec, diff --git a/migration/src/lib.rs b/migration/src/lib.rs index da1e89315..a42e3ea46 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -93,6 +93,7 @@ mod m0000720_alter_sbom_fix_null_array; mod m0000730_alter_importer_add_progress; mod m0000740_ensure_get_purl_fns; mod m0000750_alter_advisory_add_document_id; +mod m0000760_alter_sbom_alter_document_id; pub struct Migrator; @@ -193,6 +194,7 @@ impl MigratorTrait for Migrator { Box::new(m0000730_alter_importer_add_progress::Migration), Box::new(m0000740_ensure_get_purl_fns::Migration), Box::new(m0000750_alter_advisory_add_document_id::Migration), + Box::new(m0000760_alter_sbom_alter_document_id::Migration), ] } } diff --git a/migration/src/m0000760_alter_sbom_alter_document_id.rs b/migration/src/m0000760_alter_sbom_alter_document_id.rs new file mode 100644 index 000000000..f060d92b7 --- /dev/null +++ b/migration/src/m0000760_alter_sbom_alter_document_id.rs @@ -0,0 +1,55 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // modify, allow null + + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .modify_column(ColumnDef::new(Sbom::DocumentId).string().null().to_owned()) + .to_owned(), + ) + .await?; + + // bring back the null value, or consider it null if we already did not have a real value + + manager + .get_connection() + .execute_unprepared(r#"UPDATE sbom SET document_id = NULL where document_id=''"#) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // set an empty string, works and is required + + manager + .get_connection() + .execute_unprepared(r#"UPDATE sbom SET document_id = '' where document_id IS NULL"#) + .await?; + + manager + .alter_table( + Table::alter() + .table(Sbom::Table) + .modify_column(ColumnDef::new(Sbom::DocumentId).string().not_null()) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Sbom { + Table, + DocumentId, +} diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 1de9b2697..ac2b13818 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -48,7 +48,6 @@ chrono = { workspace = true } criterion = { workspace = true, features = ["html_reports", "async_tokio"] } csaf = { workspace = true } cve = { workspace = true } -cyclonedx-bom = { workspace = true } hex = { workspace = true } humantime = { workspace = true } jsonpath-rust = { workspace = true } @@ -58,11 +57,13 @@ packageurl = { workspace = true } regex = { workspace = true } roxmltree = { workspace = true } semver = { workspace = true } +serde-cyclonedx = { workspace = true } serde_json = { workspace = true } serde_yml = { workspace = true } sha2 = { workspace = true } spdx-rs = { workspace = true } strum = { workspace = true } +termimad = "0.31.0" test-context = { workspace = true } test-log = { workspace = true, features = ["log", "trace"] } tokio-util = { workspace = true } @@ -71,7 +72,6 @@ trustify-test-context = { workspace = true } urlencoding = { workspace = true } walkdir = { workspace = true } zip = { workspace = true } -termimad = "0.31.0" [[bench]] name = "bench" diff --git a/modules/fundamental/src/advisory/endpoints/test.rs b/modules/fundamental/src/advisory/endpoints/test.rs index a7b04779f..c2df6d5ab 100644 --- a/modules/fundamental/src/advisory/endpoints/test.rs +++ b/modules/fundamental/src/advisory/endpoints/test.rs @@ -349,7 +349,10 @@ async fn upload_default_csaf_format(ctx: &TrustifyContext) -> Result<(), anyhow: let result: IngestResult = app.call_and_read_body_json(request).await; log::debug!("{result:?}"); assert!(matches!(result.id, Id::Uuid(_))); - assert_eq!(result.document_id, "https://www.redhat.com/#CVE-2023-33201"); + assert_eq!( + result.document_id, + Some("https://www.redhat.com/#CVE-2023-33201".to_string()) + ); Ok(()) } @@ -407,7 +410,7 @@ async fn upload_osv_format(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let result: IngestResult = app.call_and_read_body_json(request).await; assert!(matches!(result.id, Id::Uuid(_))); - assert_eq!(result.document_id, "RUSTSEC-2021-0079"); + assert_eq!(result.document_id, Some("RUSTSEC-2021-0079".to_string())); Ok(()) } @@ -426,7 +429,7 @@ async fn upload_cve_format(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let result: IngestResult = app.call_and_read_body_json(request).await; assert!(matches!(result.id, Id::Uuid(_))); - assert_eq!(result.document_id, "CVE-2024-27088"); + assert_eq!(result.document_id, Some("CVE-2024-27088".to_string())); Ok(()) } @@ -466,7 +469,10 @@ async fn upload_with_labels(ctx: &TrustifyContext) -> Result<(), anyhow::Error> let result: IngestResult = app.call_and_read_body_json(request).await; log::debug!("{result:?}"); assert!(matches!(result.id, Id::Uuid(_))); - assert_eq!(result.document_id, "https://www.redhat.com/#CVE-2023-33201"); + assert_eq!( + result.document_id, + Some("https://www.redhat.com/#CVE-2023-33201".to_string()) + ); // now check the labels diff --git a/modules/fundamental/src/ai/service/test.rs b/modules/fundamental/src/ai/service/test.rs index 46aea6a74..0f904426e 100644 --- a/modules/fundamental/src/ai/service/test.rs +++ b/modules/fundamental/src/ai/service/test.rs @@ -13,7 +13,7 @@ pub async fn ingest_fixtures(ctx: &TrustifyContext) -> Result<(), anyhow::Error> .ingest_sbom( ("source", "http://redhat.com/test.json"), &Digests::digest("RHSA-1"), - "a", + Some("a".to_string()), (), &ctx.db, ) diff --git a/modules/fundamental/src/product/service/test.rs b/modules/fundamental/src/product/service/test.rs index 0887dfe5d..d067354a7 100644 --- a/modules/fundamental/src/product/service/test.rs +++ b/modules/fundamental/src/product/service/test.rs @@ -16,7 +16,7 @@ async fn all_products(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { .ingest_sbom( ("source", "http://redhat.com/test.json"), &Digests::digest("RHSA-1"), - "a", + Some("a".to_string()), (), &ctx.db, ) @@ -77,7 +77,7 @@ async fn link_sbom_to_product(ctx: &TrustifyContext) -> Result<(), anyhow::Error .ingest_sbom( ("source", "http://redhat.com/test.json"), &Digests::digest("RHSA-1"), - "a", + Some("a".to_string()), (), &ctx.db, ) diff --git a/modules/fundamental/src/sbom/model/mod.rs b/modules/fundamental/src/sbom/model/mod.rs index e42bb67e6..503a74535 100644 --- a/modules/fundamental/src/sbom/model/mod.rs +++ b/modules/fundamental/src/sbom/model/mod.rs @@ -20,7 +20,7 @@ pub struct SbomHead { #[schema(value_type=String)] pub id: Uuid, - pub document_id: String, + pub document_id: Option, pub labels: Labels, pub data_licenses: Vec, diff --git a/modules/fundamental/src/sbom/service/sbom.rs b/modules/fundamental/src/sbom/service/sbom.rs index d0cd746bb..04f614c06 100644 --- a/modules/fundamental/src/sbom/service/sbom.rs +++ b/modules/fundamental/src/sbom/service/sbom.rs @@ -689,7 +689,7 @@ mod test { .ingest_sbom( Labels::default(), &Digests::digest("RHSA-1"), - "http://redhat.com/test.json", + Some("http://redhat.com/test.json".to_string()), (), &ctx.db, ) @@ -699,7 +699,7 @@ mod test { .ingest_sbom( Labels::default(), &Digests::digest("RHSA-1"), - "http://redhat.com/test.json", + Some("http://redhat.com/test.json".to_string()), (), &ctx.db, ) @@ -709,7 +709,7 @@ mod test { .ingest_sbom( Labels::default(), &Digests::digest("RHSA-2"), - "http://myspace.com/test.json", + Some("http://myspace.com/test.json".to_string()), (), &ctx.db, ) @@ -720,7 +720,7 @@ mod test { .ingest_sbom( Labels::default(), &Digests::digest("RHSA-3"), - "http://geocities.com/other.json", + Some("http://geocities.com/other.json".to_string()), (), &ctx.db, ) @@ -757,7 +757,7 @@ mod test { .add("ci", "job1") .add("team", "a"), &Digests::digest("RHSA-1"), - "http://redhat.com/test1.json", + Some("http://redhat.com/test1.json".to_string()), (), &ctx.db, ) @@ -771,7 +771,7 @@ mod test { .add("ci", "job2") .add("team", "b"), &Digests::digest("RHSA-2"), - "http://redhat.com/test2.json", + Some("http://redhat.com/test2.json".to_string()), (), &ctx.db, ) @@ -785,7 +785,7 @@ mod test { .add("ci", "job2") .add("team", "a"), &Digests::digest("RHSA-3"), - "http://redhat.com/test3.json", + Some("http://redhat.com/test3.json".to_string()), (), &ctx.db, ) @@ -859,7 +859,7 @@ mod test { .ingest_sbom( Labels::default(), &Digests::digest("RHSA-1"), - "http://redhat.com/test.json", + Some("http://redhat.com/test.json".to_string()), (), &ctx.db, ) diff --git a/modules/fundamental/src/vulnerability/service/test.rs b/modules/fundamental/src/vulnerability/service/test.rs index b24e191e3..ebc0e2b2e 100644 --- a/modules/fundamental/src/vulnerability/service/test.rs +++ b/modules/fundamental/src/vulnerability/service/test.rs @@ -186,7 +186,7 @@ async fn commons_compress(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let mut sboms: Vec<_> = vuln.advisories[0] .sboms .iter() - .map(|i| i.head.document_id.as_str()) + .flat_map(|i| i.head.document_id.clone()) .collect(); sboms.sort(); @@ -267,7 +267,7 @@ async fn product_statuses(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let mut sboms: Vec<_> = vuln.advisories[0] .sboms .iter() - .map(|i| i.head.document_id.as_str()) + .flat_map(|i| i.head.document_id.clone()) .collect(); sboms.sort(); diff --git a/modules/fundamental/tests/sbom/cyclonedx/mod.rs b/modules/fundamental/tests/sbom/cyclonedx/mod.rs index 8db78e049..48bb47df4 100644 --- a/modules/fundamental/tests/sbom/cyclonedx/mod.rs +++ b/modules/fundamental/tests/sbom/cyclonedx/mod.rs @@ -97,7 +97,11 @@ where test_with( ctx, sbom, - |data| Ok(Bom::parse_from_json(data)?), + |data| { + Ok(serde_json::from_slice::< + serde_cyclonedx::cyclonedx::v_1_6::CycloneDx, + >(data)?) + }, |ctx, sbom, tx| Box::pin(async move { ctx.ingest_cyclonedx(sbom.clone(), tx).await }), |sbom| sbom::cyclonedx::Information(sbom).into(), f, diff --git a/modules/fundamental/tests/sbom/graph.rs b/modules/fundamental/tests/sbom/graph.rs index 07ddb2357..6f626264a 100644 --- a/modules/fundamental/tests/sbom/graph.rs +++ b/modules/fundamental/tests/sbom/graph.rs @@ -20,7 +20,7 @@ async fn ingest_sboms(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("8"), - "a", + Some("a".to_string()), (), &ctx.db, ) @@ -29,7 +29,7 @@ async fn ingest_sboms(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("8"), - "b", + Some("b".to_string()), (), &ctx.db, ) @@ -38,7 +38,7 @@ async fn ingest_sboms(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("9"), - "c", + Some("c".to_string()), (), &ctx.db, ) @@ -48,7 +48,7 @@ async fn ingest_sboms(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { .ingest_sbom( ("source", "http://sbom.com/other.json"), &Digests::digest("10"), - "d", + Some("d".to_string()), (), &ctx.db, ) @@ -71,7 +71,7 @@ async fn ingest_and_fetch_sboms_describing_purls( .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("8"), - "a", + Some("a".to_string()), (), &ctx.db, ) @@ -80,7 +80,7 @@ async fn ingest_and_fetch_sboms_describing_purls( .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("9"), - "b", + Some("b".to_string()), (), &ctx.db, ) @@ -89,7 +89,7 @@ async fn ingest_and_fetch_sboms_describing_purls( .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("10"), - "c", + Some("c".to_string()), (), &ctx.db, ) @@ -141,7 +141,7 @@ async fn ingest_and_locate_sboms_describing_cpes( .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("8"), - "a", + Some("a".to_string()), (), &ctx.db, ) @@ -150,7 +150,7 @@ async fn ingest_and_locate_sboms_describing_cpes( .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("9"), - "b", + Some("b".to_string()), (), &ctx.db, ) @@ -159,7 +159,7 @@ async fn ingest_and_locate_sboms_describing_cpes( .ingest_sbom( ("source", "http://sbom.com/test.json"), &Digests::digest("10"), - "c", + Some("c".to_string()), (), &ctx.db, ) @@ -200,7 +200,7 @@ async fn transitive_dependency_of(ctx: &TrustifyContext) -> Result<(), anyhow::E .ingest_sbom( ("source", "http://sbomsRus.gov/thing1.json"), &Digests::digest("8675309"), - "a", + Some("a".to_string()), (), &ctx.db, ) @@ -274,7 +274,7 @@ async fn ingest_package_relates_to_package_dependency_of( .ingest_sbom( ("source", "http://sbomsRus.gov/thing1.json"), &Digests::digest("8675309"), - "a", + Some("a".to_string()), (), &ctx.db, ) @@ -293,7 +293,7 @@ async fn ingest_package_relates_to_package_dependency_of( .ingest_sbom( ("source", "http://sbomsRus.gov/thing2.json"), &Digests::digest("8675308"), - "b", + Some("b".to_string()), (), &ctx.db, ) @@ -368,7 +368,7 @@ async fn sbom_vulnerabilities(ctx: &TrustifyContext) -> Result<(), anyhow::Error .ingest_sbom( ("source", "http://sbomsRus.gov/thing1.json"), &Digests::digest("8675309"), - "a", + Some("a".to_string()), (), &ctx.db, ) diff --git a/modules/fundamental/tests/sbom/mod.rs b/modules/fundamental/tests/sbom/mod.rs index 83fe6cd07..9cd43850d 100644 --- a/modules/fundamental/tests/sbom/mod.rs +++ b/modules/fundamental/tests/sbom/mod.rs @@ -5,7 +5,6 @@ mod graph; mod reingest; mod spdx; -use cyclonedx_bom::prelude::Bom; use sea_orm::{DatabaseTransaction, TransactionTrait}; use std::{future::Future, pin::Pin, time::Instant}; use tracing::{info_span, instrument, Instrument}; @@ -73,7 +72,7 @@ where .ingest_sbom( ("source", "test.com/my-sbom.json"), &Digests::digest("10"), - "document-id", + Some("document-id".to_string()), c(&sbom), &tx, ) diff --git a/modules/fundamental/tests/sbom/reingest.rs b/modules/fundamental/tests/sbom/reingest.rs index bd54c0903..64d517402 100644 --- a/modules/fundamental/tests/sbom/reingest.rs +++ b/modules/fundamental/tests/sbom/reingest.rs @@ -38,7 +38,7 @@ async fn quarkus(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { .ingest_document("quarkus/v1/quarkus-bom-2.13.8.Final-redhat-00004.json") .await?; - assert_eq!(result1.document_id, "https://access.redhat.com/security/data/sbom/beta/spdx/quarkus-bom-b52acd7c-3a3f-441e-aef0-bbdaa1ec8acf"); + assert_eq!(result1.document_id, Some("https://access.redhat.com/security/data/sbom/beta/spdx/quarkus-bom-b52acd7c-3a3f-441e-aef0-bbdaa1ec8acf".to_string())); // ingest the second version let result2 = ctx @@ -47,7 +47,7 @@ async fn quarkus(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result2.document_id, - "https://access.redhat.com/security/data/sbom/spdx/quarkus-bom-2.13.8.Final-redhat-00004" + Some("https://access.redhat.com/security/data/sbom/spdx/quarkus-bom-2.13.8.Final-redhat-00004".to_string()) ); // now start testing @@ -113,7 +113,7 @@ async fn nhc(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result1.document_id, - "https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8" + Some("https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8".to_string()) ); // ingest the second version @@ -121,7 +121,7 @@ async fn nhc(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result2.document_id, - "https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8" + Some("https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8".to_string()) ); // now start testing @@ -164,7 +164,7 @@ async fn nhc_same(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result1.document_id, - "https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8" + Some("https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8".to_string()) ); // ingest the same version again @@ -172,7 +172,7 @@ async fn nhc_same(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result2.document_id, - "https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8" + Some("https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8".to_string()) ); // now start testing @@ -218,7 +218,7 @@ async fn nhc_same_content(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result1.document_id, - "https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8" + Some("https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8".to_string()) ); // ingest the second version @@ -239,7 +239,7 @@ async fn nhc_same_content(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result2.document_id, - "https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8" + Some("https://access.redhat.com/security/data/sbom/spdx/RHWA-NHC-0.4-RHEL-8".to_string()) ); // now start testing @@ -288,7 +288,7 @@ async fn syft_rerun(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result1.document_id, - "https://anchore.com/syft/image/registry.access.redhat.com/ubi9/ubi-f41e17d4-e739-4d33-ab2e-48c95b856220" + Some("https://anchore.com/syft/image/registry.access.redhat.com/ubi9/ubi-f41e17d4-e739-4d33-ab2e-48c95b856220".to_string()) ); // ingest the second version @@ -296,7 +296,7 @@ async fn syft_rerun(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!( result2.document_id, - "https://anchore.com/syft/image/registry.access.redhat.com/ubi9/ubi-768a701e-12fb-4ed1-a03b-463b784b01bf" + Some("https://anchore.com/syft/image/registry.access.redhat.com/ubi9/ubi-768a701e-12fb-4ed1-a03b-463b784b01bf".to_string()) ); // now start testing diff --git a/modules/ingestor/Cargo.toml b/modules/ingestor/Cargo.toml index 5ca987930..a763cffcc 100644 --- a/modules/ingestor/Cargo.toml +++ b/modules/ingestor/Cargo.toml @@ -36,6 +36,7 @@ sea-orm = { workspace = true } sea-query = { workspace = true } semver = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde-cyclonedx = { workspace = true } serde_json = { workspace = true } serde_yml = { workspace = true } spdx = { workspace = true } diff --git a/modules/ingestor/src/graph/sbom/cyclonedx.rs b/modules/ingestor/src/graph/sbom/cyclonedx.rs index 159bd0ba0..928d88df3 100644 --- a/modules/ingestor/src/graph/sbom/cyclonedx.rs +++ b/modules/ingestor/src/graph/sbom/cyclonedx.rs @@ -8,6 +8,7 @@ use crate::graph::{ }, }; use sea_orm::ConnectionTrait; +use serde_cyclonedx::cyclonedx::v_1_6::{Component, LicenseChoiceUrl}; use std::{collections::HashMap, str::FromStr}; use time::{format_description::well_known::Iso8601, OffsetDateTime}; use tracing::instrument; @@ -22,7 +23,7 @@ use uuid::Uuid; /// component. const CYCLONEDX_DOC_REF: &str = "CycloneDX-doc-ref"; -pub struct Information<'a>(pub &'a Bom); +pub struct Information<'a>(pub &'a serde_cyclonedx::cyclonedx::v_1_6::CycloneDx); impl<'a> From> for SbomInformation { fn from(value: Information<'a>) -> Self { @@ -58,21 +59,22 @@ impl<'a> From> for SbomInformation { // otherwise use the serial number .or_else(|| sbom.serial_number.as_ref().map(|id| id.to_string())) // TODO: not sure what to use instead, the version will most likely be `1`. - .unwrap_or_else(|| sbom.version.to_string()); + .or_else(|| sbom.version.as_ref().map(|v| v.to_string())) + .unwrap_or_else(|| "".to_string()); let data_licenses = sbom .metadata .as_ref() .and_then(|metadata| metadata.licenses.as_ref()) - .map(|licenses| &licenses.0) .into_iter() - .flatten() - .map(|license| match license { - LicenseChoice::License(l) => match &l.license_identifier { - LicenseIdentifier::SpdxId(spdx) => spdx.to_string(), - LicenseIdentifier::Name(name) => name.to_string(), - }, - LicenseChoice::Expression(e) => e.to_string(), + .flat_map(|licenses| match licenses { + LicenseChoiceUrl::Variant0(license) => license + .iter() + .flat_map(|l| l.license.id.as_ref().or(l.license.name.as_ref()).cloned()) + .collect::>(), + LicenseChoiceUrl::Variant1(license) => { + license.iter().map(|l| l.expression.clone()).collect() + } }) .collect(); @@ -90,7 +92,7 @@ impl SbomContext { #[instrument(skip(connection, sbom), ret)] pub async fn ingest_cyclonedx( &self, - mut sbom: Bom, + mut sbom: serde_cyclonedx::cyclonedx::v_1_6::CycloneDx, connection: &C, ) -> Result<(), anyhow::Error> { let mut license_creator = LicenseCreator::new(); @@ -147,26 +149,38 @@ impl SbomContext { // record licenses - if let Some(components) = &sbom.components { - for component in &components.0 { - if let Some(licenses) = &component.licenses { - for license in &licenses.0 { - let license = match license { - LicenseChoice::License(license) => LicenseInfo { - license: match &license.license_identifier { - LicenseIdentifier::SpdxId(id) => id.to_string(), - LicenseIdentifier::Name(name) => name.to_string(), - }, + for component in sbom.components.iter().flatten() { + if let Some(licenses) = &component.licenses { + match licenses { + LicenseChoiceUrl::Variant0(licenses) => { + 'l: for license in licenses { + let license = if let Some(id) = license.license.id.clone() { + id + } else if let Some(name) = license.license.name.clone() { + name + } else { + continue 'l; + }; + + let license = LicenseInfo { + license, refs: Default::default(), - }, - LicenseChoice::Expression(spdx_expression) => LicenseInfo { - license: spdx_expression.to_string(), + }; + + license_creator.add(&license); + creator.add_license_relation(component, &license); + } + } + LicenseChoiceUrl::Variant1(licenses) => { + for license in licenses { + let license = LicenseInfo { + license: license.expression.clone(), refs: Default::default(), - }, - }; + }; - license_creator.add(&license); - creator.add_license_relation(component, &license); + license_creator.add(&license); + creator.add_license_relation(component, &license); + } } } } @@ -174,13 +188,9 @@ impl SbomContext { // create relationships - for left in sbom.dependencies.iter().flat_map(|e| &e.0) { - for right in &left.dependencies { - creator.relate( - right.clone(), - Relationship::DependencyOf, - left.dependency_ref.clone(), - ); + for left in sbom.dependencies.iter().flatten() { + for right in left.depends_on.iter().flatten() { + creator.relate(right.clone(), Relationship::DependencyOf, left.ref_.clone()); } } @@ -214,15 +224,13 @@ impl<'a> Creator<'a> { } } - pub fn add_all(&mut self, components: impl Into>) { - if let Some(components) = components.into() { - self.extend(&components.0) - } + pub fn add_all(&mut self, components: &'a Option>) { + self.extend(components.iter().flatten()) } pub fn add(&mut self, component: &'a Component) { self.components.push(component); - self.add_all(&component.components) + self.extend(component.components.iter().flatten()); } pub fn add_license_relation(&mut self, component: &'a Component, license: &LicenseInfo) { diff --git a/modules/ingestor/src/graph/sbom/mod.rs b/modules/ingestor/src/graph/sbom/mod.rs index 37459ffd8..2bec0418e 100644 --- a/modules/ingestor/src/graph/sbom/mod.rs +++ b/modules/ingestor/src/graph/sbom/mod.rs @@ -98,7 +98,7 @@ impl Graph { &self, labels: impl Into + Debug, digests: &Digests, - document_id: &str, + document_id: Option, info: impl Into, connection: &C, ) -> Result { @@ -132,7 +132,7 @@ impl Graph { sbom_id: Set(sbom_id), node_id: Set(node_id.clone()), - document_id: Set(document_id.to_string()), + document_id: Set(document_id), published: Set(published), authors: Set(authors), diff --git a/modules/ingestor/src/model.rs b/modules/ingestor/src/model.rs index 4b76e0aa9..f8b8a1d47 100644 --- a/modules/ingestor/src/model.rs +++ b/modules/ingestor/src/model.rs @@ -6,7 +6,7 @@ pub struct IngestResult { /// The internal ID of the document pub id: Id, /// The ID declared by the document - pub document_id: String, + pub document_id: Option, /// Warnings that occurred during the import process #[serde(default, skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, diff --git a/modules/ingestor/src/service/advisory/csaf/loader.rs b/modules/ingestor/src/service/advisory/csaf/loader.rs index 0bfa837cd..37977d3cd 100644 --- a/modules/ingestor/src/service/advisory/csaf/loader.rs +++ b/modules/ingestor/src/service/advisory/csaf/loader.rs @@ -108,7 +108,7 @@ impl<'g> CsafLoader<'g> { Ok(IngestResult { id: Id::Uuid(advisory.advisory.id), - document_id: advisory_id, + document_id: Some(advisory_id), warnings: warnings.into(), }) } diff --git a/modules/ingestor/src/service/advisory/cve/loader.rs b/modules/ingestor/src/service/advisory/cve/loader.rs index d80d26edf..284e92c58 100644 --- a/modules/ingestor/src/service/advisory/cve/loader.rs +++ b/modules/ingestor/src/service/advisory/cve/loader.rs @@ -173,7 +173,7 @@ impl<'g> CveLoader<'g> { Ok(IngestResult { id: Id::Uuid(advisory.advisory.id), - document_id: id.to_string(), + document_id: Some(id.to_string()), warnings: vec![], }) } diff --git a/modules/ingestor/src/service/advisory/osv/loader.rs b/modules/ingestor/src/service/advisory/osv/loader.rs index 520997096..84a7ee96a 100644 --- a/modules/ingestor/src/service/advisory/osv/loader.rs +++ b/modules/ingestor/src/service/advisory/osv/loader.rs @@ -169,7 +169,7 @@ impl<'g> OsvLoader<'g> { Ok(IngestResult { id: Id::Uuid(advisory.advisory.id), - document_id: osv.id, + document_id: Some(osv.id), warnings: warnings.into(), }) } diff --git a/modules/ingestor/src/service/mod.rs b/modules/ingestor/src/service/mod.rs index a749a0166..d557a5469 100644 --- a/modules/ingestor/src/service/mod.rs +++ b/modules/ingestor/src/service/mod.rs @@ -224,7 +224,7 @@ impl IngestorService { let duration = Instant::now() - start; log::debug!( - "Ingested: {} ({}): took {}", + "Ingested: {} ({:?}): took {}", result.id, result.document_id, humantime::Duration::from(duration), diff --git a/modules/ingestor/src/service/sbom/clearly_defined.rs b/modules/ingestor/src/service/sbom/clearly_defined.rs index a6e09bbaf..021daa173 100644 --- a/modules/ingestor/src/service/sbom/clearly_defined.rs +++ b/modules/ingestor/src/service/sbom/clearly_defined.rs @@ -72,7 +72,7 @@ impl<'g> ClearlyDefinedLoader<'g> { .ingest_sbom( labels, digests, - document_id, + Some(document_id.to_string()), SbomInformation { node_id: document_id.to_string(), name: document_id.to_string(), diff --git a/modules/ingestor/src/service/sbom/clearly_defined_curation.rs b/modules/ingestor/src/service/sbom/clearly_defined_curation.rs index b0d259aeb..f1c272e82 100644 --- a/modules/ingestor/src/service/sbom/clearly_defined_curation.rs +++ b/modules/ingestor/src/service/sbom/clearly_defined_curation.rs @@ -26,7 +26,13 @@ impl<'g> ClearlyDefinedCurationLoader<'g> { let sbom = self .graph - .ingest_sbom(labels, digests, &curation.document_id(), &curation, &tx) + .ingest_sbom( + labels, + digests, + Some(curation.document_id()), + &curation, + &tx, + ) .await?; sbom.ingest_clearly_defined_curation(curation, &tx) diff --git a/modules/ingestor/src/service/sbom/cyclonedx.rs b/modules/ingestor/src/service/sbom/cyclonedx.rs index db4efffcb..b4d93134a 100644 --- a/modules/ingestor/src/service/sbom/cyclonedx.rs +++ b/modules/ingestor/src/service/sbom/cyclonedx.rs @@ -3,7 +3,6 @@ use crate::{ model::IngestResult, service::Error, }; -use cyclonedx_bom::prelude::Bom; use sea_orm::TransactionTrait; use serde_json::Value; use tracing::instrument; @@ -26,13 +25,13 @@ impl<'g> CyclonedxLoader<'g> { value: Value, digests: &Digests, ) -> Result { - let sbom = Bom::parse_json_value(value) + let sbom: serde_cyclonedx::cyclonedx::v_1_6::CycloneDx = serde_json::from_value(value) .map_err(|err| Error::UnsupportedFormat(format!("Failed to parse: {err}")))?; let labels = labels.add("type", "cyclonedx"); log::info!( - "Storing - version: {}, serialNumber: {:?}", + "Storing - version: {:?}, serialNumber: {:?}", sbom.version, sbom.serial_number, ); @@ -41,16 +40,15 @@ impl<'g> CyclonedxLoader<'g> { let document_id = sbom .serial_number - .as_ref() - .map(|uuid| uuid.to_string()) - .unwrap_or_else(|| sbom.version.to_string()); + .clone() + .or_else(|| sbom.version.map(|v| v.to_string())); let ctx = self .graph .ingest_sbom( labels, digests, - &document_id, + document_id.clone(), cyclonedx::Information(&sbom), &tx, ) diff --git a/modules/ingestor/src/service/sbom/spdx.rs b/modules/ingestor/src/service/sbom/spdx.rs index 42b295e40..7b6d55396 100644 --- a/modules/ingestor/src/service/sbom/spdx.rs +++ b/modules/ingestor/src/service/sbom/spdx.rs @@ -48,7 +48,13 @@ impl<'g> SpdxLoader<'g> { let sbom = self .graph - .ingest_sbom(labels, digests, &document_id, spdx::Information(&spdx), &tx) + .ingest_sbom( + labels, + digests, + Some(document_id.clone()), + spdx::Information(&spdx), + &tx, + ) .await?; sbom.ingest_spdx(spdx, &warnings, &tx).await?; @@ -57,7 +63,7 @@ impl<'g> SpdxLoader<'g> { Ok(IngestResult { id: Id::Uuid(sbom.sbom.sbom_id), - document_id, + document_id: Some(document_id), warnings: warnings.into(), }) } diff --git a/modules/ingestor/src/service/weakness/mod.rs b/modules/ingestor/src/service/weakness/mod.rs index beceb66a4..58ada2a25 100644 --- a/modules/ingestor/src/service/weakness/mod.rs +++ b/modules/ingestor/src/service/weakness/mod.rs @@ -141,7 +141,7 @@ impl<'d> CweCatalogLoader<'d> { Ok(IngestResult { id: Id::Sha512(digests.sha512.encode_hex()), - document_id: "CWE".to_string(), + document_id: Some("CWE".to_string()), warnings: vec![], }) } diff --git a/openapi.yaml b/openapi.yaml index a80812f2e..26571a5c1 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2649,10 +2649,11 @@ components: description: The result of the ingestion process required: - id - - document_id properties: document_id: - type: string + type: + - string + - 'null' description: The ID declared by the document id: $ref: '#/components/schemas/Id' @@ -3423,7 +3424,6 @@ components: type: object required: - id - - document_id - labels - data_licenses - published @@ -3440,7 +3440,9 @@ components: items: type: string document_id: - type: string + type: + - string + - 'null' id: type: string labels: diff --git a/test-context/src/lib.rs b/test-context/src/lib.rs index 3a4f2099a..eaaf942d0 100644 --- a/test-context/src/lib.rs +++ b/test-context/src/lib.rs @@ -211,7 +211,7 @@ mod test { let ingestion_result = &result[0]; - assert!(!ingestion_result.document_id.is_empty()); + assert!(ingestion_result.document_id.is_some()); Ok(()) }