diff --git a/Cargo.lock b/Cargo.lock index 4f1ea22fc..e9633b22c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8414,15 +8414,19 @@ dependencies = [ "async-graphql", "cpe", "log", + "rstest", "schemars", "sea-orm", "serde", "serde_json", "strum 0.26.3", + "test-context", "test-log", "time", + "tokio", "trustify-common", "trustify-cvss", + "trustify-test-context", "utoipa", ] diff --git a/entity/Cargo.toml b/entity/Cargo.toml index c93836631..79f81d8ab 100644 --- a/entity/Cargo.toml +++ b/entity/Cargo.toml @@ -21,11 +21,16 @@ sea-orm = { workspace = true, features = [ ] } serde = { workspace = true } serde_json = { workspace = true } -strum = { workspace = true, features = ["derive"]} +strum = { workspace = true, features = ["derive"] } time = { workspace = true } utoipa = { workspace = true } [dev-dependencies] anyhow = { workspace = true } log = { workspace = true } +rstest = { workspace = true } +test-context = { workspace = true } test-log = { workspace = true, features = ["log", "trace"] } +tokio = { workspace = true, features = ["full"] } + +trustify-test-context = { workspace = true } diff --git a/entity/src/version_range.rs b/entity/src/version_range.rs index 369d7d25f..2201faba7 100644 --- a/entity/src/version_range.rs +++ b/entity/src/version_range.rs @@ -1,3 +1,4 @@ +use crate::version_scheme::VersionScheme; use sea_orm::entity::prelude::*; use sea_orm::sea_query::{Asterisk, Func, IntoCondition, SimpleExpr}; use trustify_common::db::VersionMatches; @@ -8,7 +9,7 @@ pub struct Model { #[sea_orm(primary_key)] pub id: Uuid, // The ID of the version scheme, which is a human-friend string key like `semver`. - pub version_scheme_id: String, + pub version_scheme_id: VersionScheme, pub low_version: Option, pub low_inclusive: Option, pub high_version: Option, diff --git a/entity/src/version_scheme.rs b/entity/src/version_scheme.rs index c6b2d3b78..78828c079 100644 --- a/entity/src/version_scheme.rs +++ b/entity/src/version_scheme.rs @@ -1,15 +1,17 @@ use sea_orm::entity::prelude::*; -#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] -#[sea_orm(table_name = "version_scheme")] -pub struct Model { - #[sea_orm(primary_key)] - pub id: String, - pub name: String, - pub description: Option, +#[derive(Copy, Clone, Eq, Hash, Debug, PartialEq, EnumIter, DeriveActiveEnum, strum::Display)] +#[sea_orm( + rs_type = "String", + db_type = "String(StringLen::None)", + rename_all = "camelCase" +)] +#[strum(serialize_all = "camelCase")] +pub enum VersionScheme { + Generic, + Git, + Semver, + Rpm, + Python, + Maven, } - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 1d22f26e0..7fe039b36 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -83,6 +83,7 @@ mod m0000631_alter_product_cpe_key; mod m0000640_create_product_status; mod m0000650_alter_advisory_tracking; mod m0000660_purl_id_indexes; +mod m0000670_version_cmp; pub struct Migrator; @@ -173,6 +174,7 @@ impl MigratorTrait for Migrator { Box::new(m0000640_create_product_status::Migration), Box::new(m0000650_alter_advisory_tracking::Migration), Box::new(m0000660_purl_id_indexes::Migration), + Box::new(m0000670_version_cmp::Migration), ] } } diff --git a/migration/src/m0000670_version_cmp.rs b/migration/src/m0000670_version_cmp.rs new file mode 100644 index 000000000..782e0abe9 --- /dev/null +++ b/migration/src/m0000670_version_cmp.rs @@ -0,0 +1,44 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +#[allow(deprecated)] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared(include_str!("m0000670_version_cmp/version_matches.sql")) + .await + .map(|_| ())?; + + manager + .get_connection() + .execute_unprepared(include_str!( + "m0000670_version_cmp/generic_version_matches.sql" + )) + .await + .map(|_| ())?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared(include_str!( + "m0000620_parallel_unsafe_pg_fns/version_matches.sql" + )) + .await + .map(|_| ())?; + + manager + .get_connection() + .execute_unprepared(r#"DROP FUNCTION generic_version_matches"#) + .await + .map(|_| ())?; + + Ok(()) + } +} diff --git a/migration/src/m0000670_version_cmp/generic_version_matches.sql b/migration/src/m0000670_version_cmp/generic_version_matches.sql new file mode 100644 index 000000000..ababd9d03 --- /dev/null +++ b/migration/src/m0000670_version_cmp/generic_version_matches.sql @@ -0,0 +1,28 @@ +-- this is just an exact version match +create or replace function generic_version_matches(version_p text, range_p version_range) + returns bool +as +$$ +begin + if range_p.low_version is not null then + if range_p.low_inclusive then + if version_p = range_p.low_version then + return true; + end if; + end if; + end if; + + if range_p.high_version is not null then + if range_p.high_inclusive then + if version_p = range_p.high_version then + return true; + end if; + end if; + end if; + + return false; + +end +$$ + language plpgsql immutable; + diff --git a/migration/src/m0000670_version_cmp/version_matches.sql b/migration/src/m0000670_version_cmp/version_matches.sql new file mode 100644 index 000000000..d5e905e5a --- /dev/null +++ b/migration/src/m0000670_version_cmp/version_matches.sql @@ -0,0 +1,42 @@ +create or replace function version_matches(version_p text, range_p version_range) + returns bool +as +$$ +declare +begin + -- for an authoritative list of support schemes, see the enum + -- `trustify_entity::version_scheme::VersionScheme` + return case + when range_p.version_scheme_id = 'git' + -- Git is git, and hard. + then gitver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'semver' + -- Semver is semver + then semver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'gem' + -- RubyGems claims to be semver + then semver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'npm' + -- NPM claims to be semver + then semver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'golang' + -- Golang claims to be semver + then semver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'nuget' + -- NuGet claims to be semver + then semver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'generic' + -- Just check if it is equal + then generic_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'rpm' + -- Look at me! I'm an RPM! I'm special! + then rpmver_version_matches(version_p, range_p) + when range_p.version_scheme_id = 'maven' + -- Look at me! I'm a Maven! I'm kinda special! + then maven_version_matches(version_p, range_p) + else + false + end; +end +$$ + language plpgsql immutable; diff --git a/modules/fundamental/src/advisory/service/test.rs b/modules/fundamental/src/advisory/service/test.rs index b9901d967..16164008a 100644 --- a/modules/fundamental/src/advisory/service/test.rs +++ b/modules/fundamental/src/advisory/service/test.rs @@ -11,6 +11,7 @@ use trustify_cvss::cvss3::{ AttackComplexity, AttackVector, Availability, Confidentiality, Cvss3Base, Integrity, PrivilegesRequired, Scope, UserInteraction, }; +use trustify_entity::version_scheme::VersionScheme; use trustify_module_ingestor::graph::advisory::{ advisory_vulnerability::{VersionInfo, VersionSpec}, AdvisoryContext, AdvisoryInformation, @@ -163,7 +164,7 @@ async fn single_advisory(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { &Purl::from_str("pkg://maven/org.apache/log4j")?, "fixed", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Maven, spec: VersionSpec::Exact("1.2.3".to_string()), }, (), @@ -176,7 +177,7 @@ async fn single_advisory(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { &Purl::from_str("pkg://maven/org.apache/log4j")?, "fixed", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Maven, spec: VersionSpec::Exact("1.2.3".to_string()), }, (), @@ -243,7 +244,7 @@ async fn delete_advisory(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { &Purl::from_str("pkg://maven/org.apache/log4j")?, "fixed", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Maven, spec: VersionSpec::Exact("1.2.3".to_string()), }, (), @@ -256,7 +257,7 @@ async fn delete_advisory(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { &Purl::from_str("pkg://maven/org.apache/log4j")?, "fixed", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Maven, spec: VersionSpec::Exact("1.2.3".to_string()), }, (), diff --git a/modules/ingestor/src/graph/advisory/advisory_vulnerability.rs b/modules/ingestor/src/graph/advisory/advisory_vulnerability.rs index 5f8de22d8..0069b90c4 100644 --- a/modules/ingestor/src/graph/advisory/advisory_vulnerability.rs +++ b/modules/ingestor/src/graph/advisory/advisory_vulnerability.rs @@ -1,20 +1,17 @@ -use crate::graph::advisory::AdvisoryContext; -use crate::graph::error::Error; -use crate::graph::vulnerability::VulnerabilityContext; +use crate::graph::{advisory::AdvisoryContext, error::Error, vulnerability::VulnerabilityContext}; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, IntoIdentity, NotSet, QueryFilter, Set}; use sea_query::{Condition, Expr, IntoCondition}; use tracing::instrument; -use trustify_common::cpe::Cpe; -use trustify_common::db::Transactional; -use trustify_common::purl::Purl; +use trustify_common::{cpe::Cpe, db::Transactional, purl::Purl}; use trustify_cvss::cvss3::Cvss3Base; -use trustify_entity as entity; -use trustify_entity::cvss3::Severity; -use trustify_entity::{purl_status, status, version_range, vulnerability}; +use trustify_entity::{ + self as entity, cvss3::Severity, purl_status, status, version_range, + version_scheme::VersionScheme, vulnerability, +}; #[derive(Clone, Eq, Hash, Debug, PartialEq)] pub struct VersionInfo { - pub scheme: String, + pub scheme: VersionScheme, pub spec: VersionSpec, } @@ -438,7 +435,9 @@ impl<'g> AdvisoryVulnerabilityContext<'g> { #[cfg(test)] mod test { - use crate::graph::advisory::advisory_vulnerability::{Version, VersionInfo, VersionSpec}; + use crate::graph::advisory::advisory_vulnerability::{ + Version, VersionInfo, VersionScheme, VersionSpec, + }; use crate::graph::Graph; use test_context::test_context; use test_log::test; @@ -474,7 +473,7 @@ mod test { &"pkg://maven/io.quarkus/quarkus-core".try_into()?, "affected", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, spec: VersionSpec::Range( Version::Inclusive("1.0.2".to_string()), Version::Exclusive("1.2.0".to_string()), @@ -490,7 +489,7 @@ mod test { &"pkg://maven/io.quarkus/quarkus-core".try_into()?, "not_affected", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, spec: VersionSpec::Exact("1.1.9".to_string()), }, Transactional::None, @@ -537,7 +536,7 @@ mod test { &"pkg://maven/io.quarkus/quarkus-core".try_into()?, "affected", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, spec: VersionSpec::Range( Version::Inclusive("1.0.2".to_string()), Version::Exclusive("1.2.0".to_string()), @@ -553,7 +552,7 @@ mod test { &"pkg://maven/io.quarkus/quarkus-core".try_into()?, "not_affected", VersionInfo { - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, spec: VersionSpec::Exact("1.1.9".to_string()), }, Transactional::None, diff --git a/modules/ingestor/src/service/advisory/csaf/creator.rs b/modules/ingestor/src/service/advisory/csaf/creator.rs index e7ae5fa87..a3cc4c2e8 100644 --- a/modules/ingestor/src/service/advisory/csaf/creator.rs +++ b/modules/ingestor/src/service/advisory/csaf/creator.rs @@ -1,23 +1,27 @@ -use crate::graph::advisory::advisory_vulnerability::Version; -use crate::graph::product::ProductInformation; -use crate::graph::Graph; -use crate::service::advisory::csaf::product_status::ProductStatus; -use crate::service::advisory::csaf::util::ResolveProductIdCache; use crate::{ graph::{ - advisory::advisory_vulnerability::{VersionInfo, VersionSpec}, + advisory::advisory_vulnerability::{Version, VersionInfo, VersionSpec}, cpe::CpeCreator, + product::ProductInformation, purl::creator::PurlCreator, + Graph, + }, + service::{ + advisory::csaf::product_status::ProductStatus, advisory::csaf::util::ResolveProductIdCache, + Error, }, - service::Error, }; use csaf::{definitions::ProductIdT, Csaf}; use sea_orm::{ActiveValue::Set, ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter}; use sea_query::IntoCondition; use std::collections::{hash_map::Entry, HashMap, HashSet}; use tracing::instrument; -use trustify_common::db::Transactional; -use trustify_common::{cpe::Cpe, db::chunk::EntityChunkedIter, purl::Purl}; +use trustify_common::{ + cpe::Cpe, + db::{chunk::EntityChunkedIter, Transactional}, + purl::Purl, +}; +use trustify_entity::version_scheme::VersionScheme; use trustify_entity::{product_status, purl_status, status, version_range}; use uuid::Uuid; @@ -167,12 +171,12 @@ impl<'a> StatusCreator<'a> { // Ingest purl status let info = match purl.version.clone() { Some(version) => VersionInfo { - scheme: "generic".to_string(), + scheme: VersionScheme::Generic, spec: VersionSpec::Exact(version), }, None => VersionInfo { spec: VersionSpec::Range(Version::Unbounded, Version::Unbounded), - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, }, }; diff --git a/modules/ingestor/src/service/advisory/csaf/product_status.rs b/modules/ingestor/src/service/advisory/csaf/product_status.rs index 31f409e06..5da548064 100644 --- a/modules/ingestor/src/service/advisory/csaf/product_status.rs +++ b/modules/ingestor/src/service/advisory/csaf/product_status.rs @@ -1,10 +1,9 @@ -use csaf::definitions::{Branch, BranchCategory, FullProductName}; -use trustify_common::purl::Purl; - +use super::util::branch_purl; use crate::graph::advisory::advisory_vulnerability::{Version, VersionInfo, VersionSpec}; use cpe::cpe::Cpe; - -use super::util::branch_purl; +use csaf::definitions::{Branch, BranchCategory, FullProductName}; +use trustify_common::purl::Purl; +use trustify_entity::version_scheme::VersionScheme; #[derive(Clone, Default, Debug, Eq, Hash, PartialEq)] pub struct ProductStatus { @@ -94,18 +93,18 @@ impl ProductStatus { Version::Inclusive(semver.to_string()), Version::Unbounded, ), - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, }, Err(_) => VersionInfo { spec: VersionSpec::Exact(version), - scheme: "generic".to_string(), + scheme: VersionScheme::Generic, }, } } else { // Treat * value as unbounded version VersionInfo { spec: VersionSpec::Range(Version::Unbounded, Version::Unbounded), - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, } } }) @@ -114,7 +113,7 @@ impl ProductStatus { // If we have purl, use an exact version purl.version().map(|version| VersionInfo { spec: VersionSpec::Exact(version.to_string()), - scheme: "semver".to_string(), + scheme: VersionScheme::Semver, }) }) }) diff --git a/modules/ingestor/src/service/advisory/cve/loader.rs b/modules/ingestor/src/service/advisory/cve/loader.rs index 1c2bb5e0d..fd35165b5 100644 --- a/modules/ingestor/src/service/advisory/cve/loader.rs +++ b/modules/ingestor/src/service/advisory/cve/loader.rs @@ -18,7 +18,7 @@ use std::fmt::Debug; use time::OffsetDateTime; use tracing::instrument; use trustify_common::{hashing::Digests, id::Id}; -use trustify_entity::labels::Labels; +use trustify_entity::{labels::Labels, version_scheme::VersionScheme}; /// Loader capable of parsing a CVE Record JSON file /// and manipulating the Graph to integrate it into @@ -142,7 +142,11 @@ impl<'g> CveLoader<'g> { Status::Unknown => "unknown", }, VersionInfo { - scheme: version_type.unwrap_or("generic".to_string()), + scheme: version_type + .and_then(|version_type| { + try_from_cve_version_scheme(&version_type).ok() + }) + .unwrap_or(VersionScheme::Generic), spec: version_spec, }, &tx, @@ -281,6 +285,30 @@ impl<'g> CveLoader<'g> { } } +/// Translate from a CVE project version type to our internal version scheme. +/// +/// Also see: +/// +/// However, the reality looks quite weird. The following command can be run to get an overview of +/// what the current state holds. Run from the `cves` directory of the repository from: +/// +/// +/// ```bash +/// find -name "CVE-*.json" -exec jq '.containers.cna.affected?[]?.versions?[]?.versionType | select (. != null )' {} \; | sort -u +/// ``` +fn try_from_cve_version_scheme(scheme: &str) -> Result { + Ok(match scheme { + "commit" | "git" => VersionScheme::Git, + "custom" => VersionScheme::Generic, + "maven" => VersionScheme::Maven, + "npm" => VersionScheme::Semver, + "python" => VersionScheme::Python, + "rpm" => VersionScheme::Rpm, + "semver" => VersionScheme::Semver, + _ => return Err(()), + }) +} + struct VulnerabilityDetails<'a> { pub org_name: Option<&'a str>, pub descriptions: &'a Vec, diff --git a/modules/ingestor/src/service/advisory/osv/loader.rs b/modules/ingestor/src/service/advisory/osv/loader.rs index 5e830cb2b..b51298115 100644 --- a/modules/ingestor/src/service/advisory/osv/loader.rs +++ b/modules/ingestor/src/service/advisory/osv/loader.rs @@ -1,22 +1,27 @@ use crate::{ graph::{ advisory::{ - advisory_vulnerability::{Version, VersionInfo, VersionSpec}, + advisory_vulnerability::{ + AdvisoryVulnerabilityContext, Version, VersionInfo, VersionSpec, + }, AdvisoryInformation, AdvisoryVulnerabilityInformation, }, purl::creator::PurlCreator, Graph, }, model::IngestResult, - service::{advisory::osv::translate, Error, Warnings}, + service::{ + advisory::osv::{prefix::get_well_known_prefixes, translate}, + Error, Warnings, + }, }; -use osv::schema::{Event, ReferenceType, SeverityType, Vulnerability}; +use osv::schema::{Event, Range, RangeType, ReferenceType, SeverityType, Vulnerability}; use sbom_walker::report::ReportSink; -use std::{fmt::Debug, str::FromStr, sync::OnceLock}; +use std::{fmt::Debug, str::FromStr}; use tracing::instrument; -use trustify_common::{hashing::Digests, id::Id, purl::Purl, time::ChronoExt}; +use trustify_common::{db::Transactional, hashing::Digests, id::Id, purl::Purl, time::ChronoExt}; use trustify_cvss::cvss3::Cvss3Base; -use trustify_entity::labels::Labels; +use trustify_entity::{labels::Labels, version_scheme::VersionScheme}; pub struct OsvLoader<'g> { graph: &'g Graph, @@ -134,54 +139,21 @@ impl<'g> OsvLoader<'g> { } for range in affected.ranges.iter().flatten() { - let parsed_range = events_to_range(&range.events); - - let spec = match &parsed_range { - (Some(start), None) => Some(VersionSpec::Range( - Version::Inclusive(start.clone()), - Version::Unbounded, - )), - (None, Some(end)) => Some(VersionSpec::Range( - Version::Unbounded, - Version::Exclusive(end.clone()), - )), - (Some(start), Some(end)) => Some(VersionSpec::Range( - Version::Inclusive(start.clone()), - Version::Exclusive(end.clone()), - )), - (None, None) => None, - }; - - if let Some(spec) = spec { - advisory_vuln - .ingest_package_status( - None, + match range.range_type { + RangeType::Semver => { + create_package_status_semver(&advisory_vuln, &purl, range, &tx) + .await?; + } + _ => { + create_package_status_versions( + &advisory_vuln, &purl, - "affected", - VersionInfo { - // TODO(#900): detect better version scheme - scheme: "semver".to_string(), - spec, - }, - &tx, - ) - .await?; - } - - if let (_, Some(fixed)) = &parsed_range { - advisory_vuln - .ingest_package_status( - None, - &purl, - "fixed", - VersionInfo { - // TODO(#900) detect better version scheme - scheme: "semver".to_string(), - spec: VersionSpec::Exact(fixed.clone()), - }, + range, + affected.versions.iter().flatten(), &tx, ) .await? + } } } } @@ -200,63 +172,199 @@ impl<'g> OsvLoader<'g> { } } -fn detect_organization(osv: &Vulnerability) -> Option { - if let Some(references) = &osv.references { - let advisory_location = references - .iter() - .find(|reference| matches!(reference.reference_type, ReferenceType::Advisory)); +/// create package statues based on listed versions +async fn create_package_status_versions( + advisory_vuln: &AdvisoryVulnerabilityContext<'_>, + purl: &Purl, + range: &Range, + versions: impl IntoIterator, + tx: impl AsRef, +) -> Result<(), Error> { + // the list of versions, sorted by the range type + let versions = versions.into_iter().cloned().collect::>(); + + let mut start = None; + for event in &range.events { + match event { + Event::Introduced(version) => { + start = Some(version); + } + Event::Fixed(version) | Event::LastAffected(version) => { + if let Some(start) = start.take() { + ingest_range_from( + advisory_vuln, + purl, + "affected", + start, + Some(version), + &versions, + &tx, + ) + .await?; + } - if let Some(advisory_location) = advisory_location { - let url = &advisory_location.url; - return get_well_known_prefixes().detect(url); + ingest_exact(advisory_vuln, purl, "fixed", version, &tx).await?; + } + Event::Limit(_) => {} + // for non_exhaustive + _ => {} } } - None -} -struct PrefixMatcher { - prefixes: Vec, + if let Some(start) = start { + ingest_range_from(advisory_vuln, purl, "affected", start, None, &versions, &tx).await?; + } + + Ok(()) } -impl PrefixMatcher { - fn new() -> Self { - Self { prefixes: vec![] } +/// Ingest all from a start to an end +async fn ingest_range_from( + advisory_vuln: &AdvisoryVulnerabilityContext<'_>, + purl: &Purl, + status: &str, + start: &str, + // exclusive end + end: Option<&str>, + versions: &[impl AsRef], + tx: impl AsRef, +) -> Result<(), Error> { + let versions = match_versions(versions, start, end); + + for version in versions { + ingest_exact(advisory_vuln, purl, status, version, &tx).await?; } - fn add(&mut self, prefix: impl Into, name: impl Into) { - self.prefixes.push(PrefixMapping { - prefix: prefix.into(), - name: name.into(), - }) - } + Ok(()) +} - fn detect(&self, input: &str) -> Option { - self.prefixes - .iter() - .find(|each| input.starts_with(&each.prefix)) - .map(|inner| inner.name.clone()) +/// Extract a list of versions according to OSV +/// +/// The idea for ECOSYSTEM and GIT is that the user provides an explicit list of versions, in the +/// right order. So we search through this list, by start and end events. Translating this into +/// exact version matches. +/// +/// See: +fn match_versions<'v>( + versions: &'v [impl AsRef], + start: &str, + end: Option<&str>, +) -> Vec<&'v str> { + let mut matches = None; + + for version in versions { + let version = version.as_ref(); + match (&mut matches, end) { + (None, _) if version == start => { + matches = Some(vec![version]); + } + (None, _) => {} + (Some(_), Some(end)) if end == version => { + // reached the exclusive env + break; + } + (Some(matches), _) => { + matches.push(version); + } + } } + + matches.unwrap_or_default() } -struct PrefixMapping { - prefix: String, - name: String, +/// Ingest an exact version +async fn ingest_exact( + advisory_vuln: &AdvisoryVulnerabilityContext<'_>, + purl: &Purl, + status: &str, + version: &str, + tx: impl AsRef, +) -> Result<(), Error> { + Ok(advisory_vuln + .ingest_package_status( + None, + purl, + status, + VersionInfo { + scheme: VersionScheme::Generic, + spec: VersionSpec::Exact(version.to_string()), + }, + &tx, + ) + .await?) } -fn get_well_known_prefixes() -> &'static PrefixMatcher { - WELL_KNOWN_PREFIXES.get_or_init(|| { - let mut matcher = PrefixMatcher::new(); +/// create a package status from a semver range +async fn create_package_status_semver( + advisory_vuln: &AdvisoryVulnerabilityContext<'_>, + purl: &Purl, + range: &Range, + tx: impl AsRef, +) -> Result<(), Error> { + let parsed_range = events_to_range(&range.events); + + let spec = match &parsed_range { + (Some(start), None) => Some(VersionSpec::Range( + Version::Inclusive(start.clone()), + Version::Unbounded, + )), + (None, Some(end)) => Some(VersionSpec::Range( + Version::Unbounded, + Version::Exclusive(end.clone()), + )), + (Some(start), Some(end)) => Some(VersionSpec::Range( + Version::Inclusive(start.clone()), + Version::Exclusive(end.clone()), + )), + (None, None) => None, + }; + + if let Some(spec) = spec { + advisory_vuln + .ingest_package_status( + None, + purl, + "affected", + VersionInfo { + scheme: VersionScheme::Semver, + spec, + }, + &tx, + ) + .await?; + } - matcher.add( - "https://rustsec.org/advisories/RUSTSEC", - "Rust Security Advisory Database", - ); + if let (_, Some(fixed)) = &parsed_range { + advisory_vuln + .ingest_package_status( + None, + purl, + "fixed", + VersionInfo { + scheme: VersionScheme::Semver, + spec: VersionSpec::Exact(fixed.clone()), + }, + &tx, + ) + .await? + } - matcher - }) + Ok(()) } -static WELL_KNOWN_PREFIXES: OnceLock = OnceLock::new(); +fn detect_organization(osv: &Vulnerability) -> Option { + if let Some(references) = &osv.references { + let advisory_location = references + .iter() + .find(|reference| matches!(reference.reference_type, ReferenceType::Advisory)); + + if let Some(advisory_location) = advisory_location { + let url = &advisory_location.url; + return get_well_known_prefixes().detect(url); + } + } + None +} fn events_to_range(events: &[Event]) -> (Option, Option) { let start = events.iter().find_map(|e| { @@ -282,6 +390,7 @@ fn events_to_range(events: &[Event]) -> (Option, Option) { mod test { use hex::ToHex; use osv::schema::Vulnerability; + use rstest::rstest; use test_context::test_context; use test_log::test; @@ -291,6 +400,8 @@ mod test { use crate::service::advisory::osv::loader::OsvLoader; + use super::*; + #[test_context(TrustifyContext)] #[test(tokio::test)] async fn loader(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { @@ -354,4 +465,15 @@ mod test { Ok(()) } + + #[rstest] + #[case("b", Some("d"), vec!["b", "c"])] + #[case("e", None, vec!["e", "f", "g"])] + #[case("x", None, vec![])] + #[case("e", Some("a"), vec!["e", "f", "g"])] + #[test_log::test] + fn test_matches(#[case] start: &str, #[case] end: Option<&str>, #[case] result: Vec<&str>) { + const INPUT: &[&str] = &["a", "b", "c", "d", "e", "f", "g"]; + assert_eq!(match_versions(INPUT, start, end), result); + } } diff --git a/modules/ingestor/src/service/advisory/osv/mod.rs b/modules/ingestor/src/service/advisory/osv/mod.rs index 9b3442270..4afdf1516 100644 --- a/modules/ingestor/src/service/advisory/osv/mod.rs +++ b/modules/ingestor/src/service/advisory/osv/mod.rs @@ -1,3 +1,5 @@ +mod prefix; + pub mod loader; pub mod translate; diff --git a/modules/ingestor/src/service/advisory/osv/prefix.rs b/modules/ingestor/src/service/advisory/osv/prefix.rs new file mode 100644 index 000000000..3e7091519 --- /dev/null +++ b/modules/ingestor/src/service/advisory/osv/prefix.rs @@ -0,0 +1,45 @@ +use std::sync::OnceLock; + +pub struct PrefixMatcher { + prefixes: Vec, +} + +impl PrefixMatcher { + fn new() -> Self { + Self { prefixes: vec![] } + } + + fn add(&mut self, prefix: impl Into, name: impl Into) { + self.prefixes.push(PrefixMapping { + prefix: prefix.into(), + name: name.into(), + }) + } + + pub fn detect(&self, input: &str) -> Option { + self.prefixes + .iter() + .find(|each| input.starts_with(&each.prefix)) + .map(|inner| inner.name.clone()) + } +} + +struct PrefixMapping { + prefix: String, + name: String, +} + +pub fn get_well_known_prefixes() -> &'static PrefixMatcher { + WELL_KNOWN_PREFIXES.get_or_init(|| { + let mut matcher = PrefixMatcher::new(); + + matcher.add( + "https://rustsec.org/advisories/RUSTSEC", + "Rust Security Advisory Database", + ); + + matcher + }) +} + +static WELL_KNOWN_PREFIXES: OnceLock = OnceLock::new(); diff --git a/modules/ingestor/tests/ingestor.rs b/modules/ingestor/tests/ingestor.rs index b322017cb..5f5211752 100644 --- a/modules/ingestor/tests/ingestor.rs +++ b/modules/ingestor/tests/ingestor.rs @@ -1,2 +1,3 @@ mod performance; mod reingest; +mod version; diff --git a/migration/tests/version_common.rs b/modules/ingestor/tests/version/common.rs similarity index 63% rename from migration/tests/version_common.rs rename to modules/ingestor/tests/version/common.rs index 6c3a17362..021771dad 100644 --- a/migration/tests/version_common.rs +++ b/modules/ingestor/tests/version/common.rs @@ -1,26 +1,53 @@ -use migration::sea_orm::Statement; -use migration::ConnectionTrait; +#![allow(unused)] +// clippy complains about this module being imported multiple times. However, that seems to be some +// artifact from the way the group integration tests. +#![allow(clippy::duplicate_mod)] + +use sea_orm::{ConnectionTrait, Statement}; +use std::fmt::{Debug, Display}; +use std::ops::{Bound, Range, RangeBounds}; +use test_context::test_context; +use tracing::instrument; use trustify_common::db::Database; +use trustify_test_context::TrustifyContext; -#[allow(unused)] +#[derive(Debug)] pub enum VersionRange { Exact(&'static str), Range(Version, Version), } -#[allow(unused)] +impl VersionRange { + pub fn range(range: impl RangeBounds<&'static str>) -> Self { + let start = range.start_bound().map(|s| *s).into(); + let end = range.end_bound().map(|s| *s).into(); + Self::Range(start, end) + } +} + +impl From> for Version { + fn from(value: Bound<&'static str>) -> Self { + match value { + Bound::Unbounded => Version::Unbounded, + Bound::Included(version) => Version::Inclusive(version), + Bound::Excluded(version) => Version::Exclusive(version), + } + } +} + +#[derive(Debug)] pub enum Version { Inclusive(&'static str), Exclusive(&'static str), Unbounded, } -#[allow(unused)] +#[instrument(skip(db), ret)] pub async fn version_matches( db: &Database, candidate: &str, range: VersionRange, - version_scheme: &str, + version_scheme: impl Display + Debug, ) -> Result { let (low, low_inclusive, high, high_inclusive) = match range { VersionRange::Exact(version) => ( diff --git a/migration/tests/mavenver.rs b/modules/ingestor/tests/version/mavenver.rs similarity index 97% rename from migration/tests/mavenver.rs rename to modules/ingestor/tests/version/mavenver.rs index c69f24952..e0851692a 100644 --- a/migration/tests/mavenver.rs +++ b/modules/ingestor/tests/version/mavenver.rs @@ -1,13 +1,12 @@ -use crate::version_common::{version_matches, Version, VersionRange}; -use migration::sea_orm::Statement; -use migration::ConnectionTrait; +use crate::version::common::{version_matches, Version, VersionRange}; +use sea_orm::{ConnectionTrait, Statement}; use test_context::test_context; use test_log::test; use trustify_common::db::Database; use trustify_test_context::TrustifyContext; -#[path = "./version_common.rs"] -mod version_common; +#[path = "common.rs"] +mod common; async fn mavenver_cmp( db: &Database, diff --git a/modules/ingestor/tests/version/mod.rs b/modules/ingestor/tests/version/mod.rs new file mode 100644 index 000000000..2595c10b1 --- /dev/null +++ b/modules/ingestor/tests/version/mod.rs @@ -0,0 +1,33 @@ +mod common; +mod mavenver; +mod rpmver; +mod semver; + +use crate::version::common::{version_matches, VersionRange}; +use rstest::rstest; +use test_context::AsyncTestContext; +use trustify_entity::version_scheme::VersionScheme; +use trustify_test_context::TrustifyContext; + +#[rstest] +#[case("1", VersionRange::Exact("1"), VersionScheme::Generic, true)] +#[case("1.0", VersionRange::Exact("1"), VersionScheme::Generic, false)] +#[case("1.0.0", VersionRange::Exact("1.0.0"), VersionScheme::Semver, true)] +#[case("1.0.1", VersionRange::Exact("1.0.0"), VersionScheme::Semver, false)] +#[case("1.0.1", VersionRange::range("1".."2"), VersionScheme::Semver, true)] +#[case("1.0.1", VersionRange::range("1".."1.2"), VersionScheme::Semver, true)] +#[case("1.0.1", VersionRange::range("1".."1.0.2"), VersionScheme::Semver, true)] +#[test_log::test(tokio::test)] +async fn versions( + #[case] candidate: &str, + #[case] range: VersionRange, + #[case] version_scheme: VersionScheme, + #[case] expected: bool, +) -> anyhow::Result<()> { + let ctx = TrustifyContext::setup().await; + + let actual = version_matches(&ctx.db, candidate, range, version_scheme).await?; + assert_eq!(actual, expected); + + Ok(()) +} diff --git a/migration/tests/rpmver.rs b/modules/ingestor/tests/version/rpmver.rs similarity index 89% rename from migration/tests/rpmver.rs rename to modules/ingestor/tests/version/rpmver.rs index a458d4b48..b78afb204 100644 --- a/migration/tests/rpmver.rs +++ b/modules/ingestor/tests/version/rpmver.rs @@ -1,12 +1,11 @@ -use migration::sea_orm::Statement; -use migration::ConnectionTrait; +use sea_orm::{ConnectionTrait, Statement}; use test_context::test_context; use test_log::test; use trustify_common::db::Database; use trustify_test_context::TrustifyContext; -#[path = "./version_common.rs"] -mod version_common; +#[path = "common.rs"] +mod common; async fn rpmver_cmp(db: &Database, left: &str, right: &str) -> Result, anyhow::Error> { let result = db diff --git a/migration/tests/semver.rs b/modules/ingestor/tests/version/semver.rs similarity index 97% rename from migration/tests/semver.rs rename to modules/ingestor/tests/version/semver.rs index 3cb7fa822..80c2ee1e3 100644 --- a/migration/tests/semver.rs +++ b/modules/ingestor/tests/version/semver.rs @@ -1,13 +1,12 @@ -use crate::version_common::{version_matches, Version, VersionRange}; -use migration::sea_orm::Statement; -use migration::ConnectionTrait; +use crate::version::common::{version_matches, Version, VersionRange}; +use sea_orm::{ConnectionTrait, Statement}; use test_context::test_context; use test_log::test; use trustify_common::db::Database; use trustify_test_context::TrustifyContext; -#[path = "./version_common.rs"] -mod version_common; +#[path = "common.rs"] +mod common; async fn semver_cmp(db: &Database, left: &str, right: &str) -> Result, anyhow::Error> { let result = db