feat: allow processing CDX 1.6

trustification · Dec 5, 2024 · c14b342 · c14b342
1 parent 464de8e
commit c14b342
Show file tree

Hide file tree

Showing 31 changed files with 267 additions and 113 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -115,6 +115,7 @@ sea-orm-migration = "1"
 sea-query = "0.32.0"
 semver = "1"
 serde = "1.0.183"
+serde-cyclonedx = "0.9.1"
 serde_json = "1.0.114"
 serde_with = "3.11.0"
 serde_yml = "0.0.12"

diff --git a/entity/src/sbom.rs b/entity/src/sbom.rs
@@ -12,7 +12,7 @@ pub struct Model {
     pub sbom_id: Uuid,
     pub node_id: String,
 
-    pub document_id: String,
+    pub document_id: Option<String>,
 
     pub published: Option<OffsetDateTime>,
     pub authors: Vec<String>,

diff --git a/migration/src/lib.rs b/migration/src/lib.rs
@@ -93,6 +93,7 @@ mod m0000720_alter_sbom_fix_null_array;
 mod m0000730_alter_importer_add_progress;
 mod m0000740_ensure_get_purl_fns;
 mod m0000750_alter_advisory_add_document_id;
+mod m0000760_alter_sbom_alter_document_id;
 
 pub struct Migrator;
 
@@ -193,6 +194,7 @@ impl MigratorTrait for Migrator {
             Box::new(m0000730_alter_importer_add_progress::Migration),
             Box::new(m0000740_ensure_get_purl_fns::Migration),
             Box::new(m0000750_alter_advisory_add_document_id::Migration),
+            Box::new(m0000760_alter_sbom_alter_document_id::Migration),
         ]
     }
 }

diff --git a/migration/src/m0000760_alter_sbom_alter_document_id.rs b/migration/src/m0000760_alter_sbom_alter_document_id.rs
@@ -0,0 +1,55 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // modify, allow null
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Sbom::Table)
+                    .modify_column(ColumnDef::new(Sbom::DocumentId).string().null().to_owned())
+                    .to_owned(),
+            )
+            .await?;
+
+        // bring back the null value, or consider it null if we already did not have a real value
+
+        manager
+            .get_connection()
+            .execute_unprepared(r#"UPDATE sbom SET document_id = NULL where document_id=''"#)
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // set an empty string, works and is required
+
+        manager
+            .get_connection()
+            .execute_unprepared(r#"UPDATE sbom SET document_id = '' where document_id IS NULL"#)
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Sbom::Table)
+                    .modify_column(ColumnDef::new(Sbom::DocumentId).string().not_null())
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum Sbom {
+    Table,
+    DocumentId,
+}
diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml
@@ -48,7 +48,6 @@ chrono = { workspace = true }
 criterion = { workspace = true, features = ["html_reports", "async_tokio"] }
 csaf = { workspace = true }
 cve = { workspace = true }
-cyclonedx-bom = { workspace = true }
 hex = { workspace = true }
 humantime = { workspace = true }
 jsonpath-rust = { workspace = true }
@@ -58,11 +57,13 @@ packageurl = { workspace = true }
 regex = { workspace = true }
 roxmltree = { workspace = true }
 semver = { workspace = true }
+serde-cyclonedx = { workspace = true }
 serde_json = { workspace = true }
 serde_yml = { workspace = true }
 sha2 = { workspace = true }
 spdx-rs = { workspace = true }
 strum = { workspace = true }
+termimad = "0.31.0"
 test-context = { workspace = true }
 test-log = { workspace = true, features = ["log", "trace"] }
 tokio-util = { workspace = true }
@@ -71,7 +72,6 @@ trustify-test-context = { workspace = true }
 urlencoding = { workspace = true }
 walkdir = { workspace = true }
 zip = { workspace = true }
-termimad = "0.31.0"
 
 [[bench]]
 name = "bench"

diff --git a/modules/fundamental/src/advisory/endpoints/test.rs b/modules/fundamental/src/advisory/endpoints/test.rs
@@ -349,7 +349,10 @@ async fn upload_default_csaf_format(ctx: &TrustifyContext) -> Result<(), anyhow:
     let result: IngestResult = app.call_and_read_body_json(request).await;
     log::debug!("{result:?}");
     assert!(matches!(result.id, Id::Uuid(_)));
-    assert_eq!(result.document_id, "https://www.redhat.com/#CVE-2023-33201");
+    assert_eq!(
+        result.document_id,
+        Some("https://www.redhat.com/#CVE-2023-33201".to_string())
+    );
 
     Ok(())
 }
@@ -407,7 +410,7 @@ async fn upload_osv_format(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {
 
     let result: IngestResult = app.call_and_read_body_json(request).await;
     assert!(matches!(result.id, Id::Uuid(_)));
-    assert_eq!(result.document_id, "RUSTSEC-2021-0079");
+    assert_eq!(result.document_id, Some("RUSTSEC-2021-0079".to_string()));
 
     Ok(())
 }
@@ -426,7 +429,7 @@ async fn upload_cve_format(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {
 
     let result: IngestResult = app.call_and_read_body_json(request).await;
     assert!(matches!(result.id, Id::Uuid(_)));
-    assert_eq!(result.document_id, "CVE-2024-27088");
+    assert_eq!(result.document_id, Some("CVE-2024-27088".to_string()));
 
     Ok(())
 }
@@ -466,7 +469,10 @@ async fn upload_with_labels(ctx: &TrustifyContext) -> Result<(), anyhow::Error>
     let result: IngestResult = app.call_and_read_body_json(request).await;
     log::debug!("{result:?}");
     assert!(matches!(result.id, Id::Uuid(_)));
-    assert_eq!(result.document_id, "https://www.redhat.com/#CVE-2023-33201");
+    assert_eq!(
+        result.document_id,
+        Some("https://www.redhat.com/#CVE-2023-33201".to_string())
+    );
 
     // now check the labels
 

diff --git a/modules/fundamental/src/ai/service/test.rs b/modules/fundamental/src/ai/service/test.rs
@@ -13,7 +13,7 @@ pub async fn ingest_fixtures(ctx: &TrustifyContext) -> Result<(), anyhow::Error>
         .ingest_sbom(
             ("source", "http://redhat.com/test.json"),
             &Digests::digest("RHSA-1"),
-            "a",
+            Some("a".to_string()),
             (),
             &ctx.db,
         )

diff --git a/modules/fundamental/src/product/service/test.rs b/modules/fundamental/src/product/service/test.rs
@@ -16,7 +16,7 @@ async fn all_products(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {
         .ingest_sbom(
             ("source", "http://redhat.com/test.json"),
             &Digests::digest("RHSA-1"),
-            "a",
+            Some("a".to_string()),
             (),
             &ctx.db,
         )
@@ -77,7 +77,7 @@ async fn link_sbom_to_product(ctx: &TrustifyContext) -> Result<(), anyhow::Error
         .ingest_sbom(
             ("source", "http://redhat.com/test.json"),
             &Digests::digest("RHSA-1"),
-            "a",
+            Some("a".to_string()),
             (),
             &ctx.db,
         )

diff --git a/modules/fundamental/src/sbom/model/mod.rs b/modules/fundamental/src/sbom/model/mod.rs
@@ -20,7 +20,7 @@ pub struct SbomHead {
     #[schema(value_type=String)]
     pub id: Uuid,
 
-    pub document_id: String,
+    pub document_id: Option<String>,
     pub labels: Labels,
     pub data_licenses: Vec<String>,
 

diff --git a/modules/fundamental/src/sbom/service/sbom.rs b/modules/fundamental/src/sbom/service/sbom.rs
@@ -689,7 +689,7 @@ mod test {
             .ingest_sbom(
                 Labels::default(),
                 &Digests::digest("RHSA-1"),
-                "http://redhat.com/test.json",
+                Some("http://redhat.com/test.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -699,7 +699,7 @@ mod test {
             .ingest_sbom(
                 Labels::default(),
                 &Digests::digest("RHSA-1"),
-                "http://redhat.com/test.json",
+                Some("http://redhat.com/test.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -709,7 +709,7 @@ mod test {
             .ingest_sbom(
                 Labels::default(),
                 &Digests::digest("RHSA-2"),
-                "http://myspace.com/test.json",
+                Some("http://myspace.com/test.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -720,7 +720,7 @@ mod test {
             .ingest_sbom(
                 Labels::default(),
                 &Digests::digest("RHSA-3"),
-                "http://geocities.com/other.json",
+                Some("http://geocities.com/other.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -757,7 +757,7 @@ mod test {
                     .add("ci", "job1")
                     .add("team", "a"),
                 &Digests::digest("RHSA-1"),
-                "http://redhat.com/test1.json",
+                Some("http://redhat.com/test1.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -771,7 +771,7 @@ mod test {
                     .add("ci", "job2")
                     .add("team", "b"),
                 &Digests::digest("RHSA-2"),
-                "http://redhat.com/test2.json",
+                Some("http://redhat.com/test2.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -785,7 +785,7 @@ mod test {
                     .add("ci", "job2")
                     .add("team", "a"),
                 &Digests::digest("RHSA-3"),
-                "http://redhat.com/test3.json",
+                Some("http://redhat.com/test3.json".to_string()),
                 (),
                 &ctx.db,
             )
@@ -859,7 +859,7 @@ mod test {
             .ingest_sbom(
                 Labels::default(),
                 &Digests::digest("RHSA-1"),
-                "http://redhat.com/test.json",
+                Some("http://redhat.com/test.json".to_string()),
                 (),
                 &ctx.db,
             )