From 9ef04668993deb3cd9345391848aed85cd43af11 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 2 Dec 2024 15:18:50 +1100 Subject: [PATCH 01/12] feat(config): add simplified config parsing --- htsget-config/Cargo.toml | 2 +- htsget-config/src/config/data_server.rs | 56 +++++++++++++++++++ htsget-config/src/config/location.rs | 66 +++++++++++++++++++++++ htsget-config/src/config/mod.rs | 5 ++ htsget-config/src/config/service_info.rs | 50 +++++++++++++++++ htsget-config/src/config/simple.rs | 55 +++++++++++++++++++ htsget-config/src/config/ticket_server.rs | 42 +++++++++++++++ 7 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 htsget-config/src/config/data_server.rs create mode 100644 htsget-config/src/config/location.rs create mode 100644 htsget-config/src/config/service_info.rs create mode 100644 htsget-config/src/config/simple.rs create mode 100644 htsget-config/src/config/ticket_server.rs diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 00e7f3ba..5b3a3c1c 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -22,6 +22,7 @@ async-trait = "0.1" noodles = { version = "0.83", features = ["core"] } serde = { version = "1", features = ["derive"] } serde_with = "3" +serde_json = "1" serde_regex = "1" regex = "1" figment = { version = "0.10", features = ["env", "toml"] } @@ -50,7 +51,6 @@ aws-config = { version = "1", optional = true } tempfile = { version = "3", optional = true } [dev-dependencies] -serde_json = "1" figment = { version = "0.10", features = ["test"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } tempfile = "3" diff --git a/htsget-config/src/config/data_server.rs b/htsget-config/src/config/data_server.rs new file mode 100644 index 00000000..f6f71041 --- /dev/null +++ b/htsget-config/src/config/data_server.rs @@ -0,0 +1,56 @@ +//! Data server configuration. +//! + +use crate::config::{default_localstorage_addr, default_path}; +use crate::tls::TlsServerConfig; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +/// Configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct DataServerConfig { + addr: SocketAddr, + local_path: PathBuf, + #[serde(skip_serializing)] + tls: Option, +} + +impl DataServerConfig { + /// Create the ticket server config. + pub fn new(addr: SocketAddr, local_path: PathBuf, tls: Option) -> Self { + Self { + addr, + local_path, + tls, + } + } + + /// Get the socket address. + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Get the local path. + pub fn local_path(&self) -> &Path { + self.local_path.as_path() + } + + /// Get the TLS config. + pub fn tls(&self) -> Option<&TlsServerConfig> { + self.tls.as_ref() + } +} + +impl Default for DataServerConfig { + fn default() -> Self { + Self { + addr: default_localstorage_addr() + .parse() + .expect("expected valid address"), + local_path: default_path().into(), + tls: None, + } + } +} diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs new file mode 100644 index 00000000..c0437e05 --- /dev/null +++ b/htsget-config/src/config/location.rs @@ -0,0 +1,66 @@ +//! Location configuration. +//! + +use crate::config::location::Endpoint::{File, Url, S3}; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize}; + +/// Location config. +#[derive(Serialize, Debug, Clone, Default)] +#[serde(default)] +pub struct Location { + endpoint: Endpoint, + prefix: String, +} + +impl Location { + /// Create a new location. + pub fn new(endpoint: Endpoint, prefix: String) -> Self { + Self { endpoint, prefix } + } + + /// Get the endpoint. + pub fn endpoint(&self) -> Endpoint { + self.endpoint + } + + /// Get the prefix. + pub fn prefix(&self) -> &str { + &self.prefix + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, Copy)] +pub enum Endpoint { + #[default] + File, + S3, + Url, +} + +impl<'de> Deserialize<'de> for Location { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?.to_lowercase(); + + let endpoint = if s.strip_prefix("file://").is_some() { + File + } else if s.strip_prefix("s3://").is_some() { + S3 + } else if s + .strip_prefix("http://") + .or_else(|| s.strip_prefix("https://")) + .is_some() + { + Url + } else { + return Err(Error::custom( + "expected file://, s3://, http:// or https:// scheme", + )); + }; + + Ok(Location::new(endpoint, s)) + } +} diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 71750fe9..0a0d8add 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -23,7 +23,12 @@ use crate::resolver::Resolver; use crate::tls::TlsServerConfig; pub mod cors; +pub mod data_server; +mod location; pub mod parser; +pub mod service_info; +pub mod simple; +pub mod ticket_server; /// Represents a usage string for htsget-rs. pub const USAGE: &str = "To configure htsget-rs use a config file or environment variables. \ diff --git a/htsget-config/src/config/service_info.rs b/htsget-config/src/config/service_info.rs new file mode 100644 index 00000000..0e7eafdc --- /dev/null +++ b/htsget-config/src/config/service_info.rs @@ -0,0 +1,50 @@ +//! Service info configuration. +//! + +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +/// Service info config. +#[derive(Serialize, Debug, Clone, Default)] +#[serde(default)] +pub struct ServiceInfo { + fields: HashMap, +} + +impl ServiceInfo { + /// Create a service info. + pub fn new(fields: HashMap) -> Self { + Self { fields } + } + + /// Get the inner value. + pub fn into_inner(self) -> HashMap { + self.fields + } +} + +impl<'de> Deserialize<'de> for ServiceInfo { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let fields: HashMap = HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| (key.to_lowercase(), value)) + .collect(); + + let err_msg = |invalid_key| format!("reserved service info field `{}`", invalid_key); + + if fields.contains_key("type") { + return Err(Error::custom(err_msg("type"))); + } + + if fields.contains_key("htsget") { + return Err(Error::custom(err_msg("htsget"))); + } + + Ok(ServiceInfo::new(fields)) + } +} diff --git a/htsget-config/src/config/simple.rs b/htsget-config/src/config/simple.rs new file mode 100644 index 00000000..5fb95e01 --- /dev/null +++ b/htsget-config/src/config/simple.rs @@ -0,0 +1,55 @@ +//! Simplified configuration. +//! + +use crate::config::data_server::DataServerConfig; +use crate::config::location::Location; +use crate::config::service_info::ServiceInfo; +use crate::config::ticket_server::TicketServerConfig; +use serde::{Deserialize, Serialize}; + +/// Simplified config. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct Config { + ticket_server: TicketServerConfig, + data_server: Option, + service_info: Option, + location: Option, +} + +impl Config { + /// Create a config. + pub fn new( + ticket_server: TicketServerConfig, + data_server: Option, + service_info: Option, + location: Option, + ) -> Self { + Self { + ticket_server, + data_server, + service_info, + location, + } + } + + /// Get the ticket server config. + pub fn ticket_server(&self) -> &TicketServerConfig { + &self.ticket_server + } + + /// Get the data server config. + pub fn data_server(&self) -> Option<&DataServerConfig> { + self.data_server.as_ref() + } + + /// Get the service info config. + pub fn service_info(&self) -> Option<&ServiceInfo> { + self.service_info.as_ref() + } + + /// Get the location. + pub fn location(&self) -> Option<&Location> { + self.location.as_ref() + } +} diff --git a/htsget-config/src/config/ticket_server.rs b/htsget-config/src/config/ticket_server.rs new file mode 100644 index 00000000..ce6db032 --- /dev/null +++ b/htsget-config/src/config/ticket_server.rs @@ -0,0 +1,42 @@ +//! Ticket server configuration. +//! + +use crate::config::default_addr; +use crate::tls::TlsServerConfig; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; + +/// Configuration for the htsget ticket server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct TicketServerConfig { + addr: SocketAddr, + #[serde(skip_serializing)] + tls: Option, +} + +impl TicketServerConfig { + /// Create the ticket server config. + pub fn new(addr: SocketAddr, tls: Option) -> Self { + Self { addr, tls } + } + + /// Get the socket address. + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Get the TLS config. + pub fn tls(&self) -> Option<&TlsServerConfig> { + self.tls.as_ref() + } +} + +impl Default for TicketServerConfig { + fn default() -> Self { + Self { + addr: default_addr().parse().expect("expected valid address"), + tls: None, + } + } +} From 4b46fab25ada9264f3ad11342fe6f504b0de8e0f Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Dec 2024 09:55:06 +1100 Subject: [PATCH 02/12] feat(config): move cors to advanced and set defaults to mirror as suggested by spec --- htsget-config/src/config/advanced/cors.rs | 296 ++++++++++++++++++++++ htsget-config/src/config/advanced/mod.rs | 4 + htsget-config/src/config/data_server.rs | 18 +- htsget-config/src/config/mod.rs | 3 +- htsget-config/src/config/ticket_server.rs | 14 +- 5 files changed, 329 insertions(+), 6 deletions(-) create mode 100644 htsget-config/src/config/advanced/cors.rs create mode 100644 htsget-config/src/config/advanced/mod.rs diff --git a/htsget-config/src/config/advanced/cors.rs b/htsget-config/src/config/advanced/cors.rs new file mode 100644 index 00000000..901cf384 --- /dev/null +++ b/htsget-config/src/config/advanced/cors.rs @@ -0,0 +1,296 @@ +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +use http::header::{HeaderName, HeaderValue as HeaderValueInner, InvalidHeaderValue}; +use http::Method; +use serde::de::Error; +use serde::ser::SerializeSeq; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::types::TaggedTypeAll; + +/// The maximum default amount of time a CORS request can be cached for in seconds. +/// Defaults to 30 days. +const CORS_MAX_AGE: usize = 2592000; + +/// Tagged allow headers for cors config, either Mirror or Any. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum TaggedAllowTypes { + #[serde(alias = "mirror", alias = "MIRROR")] + Mirror, + #[serde(alias = "all", alias = "ALL")] + All, +} + +/// Allowed type for cors config which is used to configure cors behaviour. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(untagged)] +pub enum AllowType { + Tagged(Tagged), + #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] + #[serde( + serialize_with = "serialize_allow_types", + deserialize_with = "deserialize_allow_types" + )] + List(Vec), +} + +impl AllowType { + /// Apply a function to the builder when the type is a List. + pub fn apply_list(&self, func: F, builder: U) -> U + where + F: FnOnce(U, &Vec) -> U, + { + if let Self::List(list) = self { + func(builder, list) + } else { + builder + } + } + + /// Apply a function to the builder when the type is a List returning a Result. + pub fn try_apply_list(&self, func: F, builder: U) -> Result + where + F: FnOnce(U, &Vec) -> Result, + { + if let Self::List(list) = self { + func(builder, list) + } else { + Ok(builder) + } + } + + /// Apply a function to the builder when the type is tagged. + pub fn apply_tagged(&self, func: F, builder: U, tagged_type: &Tagged) -> U + where + F: FnOnce(U) -> U, + Tagged: Eq, + { + if let Self::Tagged(tagged) = self { + if tagged == tagged_type { + return func(builder); + } + } + + builder + } +} + +impl AllowType { + /// Apply a function to the builder when the type is Mirror. + pub fn apply_mirror(&self, func: F, builder: U) -> U + where + F: FnOnce(U) -> U, + { + self.apply_tagged(func, builder, &TaggedAllowTypes::Mirror) + } + + /// Apply a function to the builder when the type is Any. + pub fn apply_any(&self, func: F, builder: U) -> U + where + F: FnOnce(U) -> U, + { + self.apply_tagged(func, builder, &TaggedAllowTypes::All) + } +} + +impl AllowType { + /// Apply a function to the builder when the type is Any. + pub fn apply_any(&self, func: F, builder: U) -> U + where + F: FnOnce(U) -> U, + { + self.apply_tagged(func, builder, &TaggedTypeAll::All) + } +} + +fn serialize_allow_types(names: &[T], serializer: S) -> Result +where + T: Display, + S: Serializer, +{ + let mut sequence = serializer.serialize_seq(Some(names.len()))?; + for element in names.iter().map(|name| format!("{name}")) { + sequence.serialize_element(&element)?; + } + sequence.end() +} + +fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Error> +where + T: FromStr, + T::Err: Display, + D: Deserializer<'de>, +{ + let names: Vec = Deserialize::deserialize(deserializer)?; + names + .into_iter() + .map(|name| T::from_str(&name).map_err(Error::custom)) + .collect() +} + +/// A wrapper around a http HeaderValue which is used to implement FromStr and Display. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HeaderValue(HeaderValueInner); + +impl HeaderValue { + pub fn into_inner(self) -> HeaderValueInner { + self.0 + } +} + +impl FromStr for HeaderValue { + type Err = InvalidHeaderValue; + + fn from_str(header: &str) -> Result { + Ok(HeaderValue(HeaderValueInner::from_str(header)?)) + } +} + +impl Display for HeaderValue { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&String::from_utf8_lossy(self.0.as_ref())) + } +} + +/// Cors configuration for the htsget server. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct CorsConfig { + allow_credentials: bool, + allow_origins: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, + max_age: usize, + expose_headers: AllowType, +} + +impl CorsConfig { + /// Create new cors config. + pub fn new( + allow_credentials: bool, + allow_origins: AllowType, + allow_headers: AllowType, + allow_methods: AllowType, + max_age: usize, + expose_headers: AllowType, + ) -> Self { + Self { + allow_credentials, + allow_origins, + allow_headers, + allow_methods, + max_age, + expose_headers, + } + } + + /// Get allow credentials. + pub fn allow_credentials(&self) -> bool { + self.allow_credentials + } + + /// Get allow origins. + pub fn allow_origins(&self) -> &AllowType { + &self.allow_origins + } + + /// Get allow headers. + pub fn allow_headers(&self) -> &AllowType { + &self.allow_headers + } + + /// Get allow methods. + pub fn allow_methods(&self) -> &AllowType { + &self.allow_methods + } + + /// Get max age. + pub fn max_age(&self) -> usize { + self.max_age + } + + /// Get expose headers. + pub fn expose_headers(&self) -> &AllowType { + &self.expose_headers + } +} + +impl Default for CorsConfig { + fn default() -> Self { + Self { + allow_credentials: false, + allow_origins: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), + max_age: CORS_MAX_AGE, + expose_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + } + } +} + +#[cfg(test)] +mod tests { + use std::fmt::Debug; + + use http::Method; + use toml::de::Error; + + use super::*; + + fn test_cors_config(input: &str, expected: &T, get_result: F) + where + F: Fn(&CorsConfig) -> &T, + T: Debug + Eq, + { + let config: CorsConfig = toml::from_str(input).unwrap(); + assert_eq!(expected, get_result(&config)); + + let serialized = toml::to_string(&config).unwrap(); + let deserialized = toml::from_str(&serialized).unwrap(); + assert_eq!(expected, get_result(&deserialized)); + } + + #[test] + fn unit_variant_any_allow_type() { + test_cors_config( + "allow_methods = \"All\"", + &AllowType::Tagged(TaggedAllowTypes::All), + |config| config.allow_methods(), + ); + } + + #[test] + fn unit_variant_mirror_allow_type() { + test_cors_config( + "allow_origins = \"Mirror\"", + &AllowType::Tagged(TaggedAllowTypes::Mirror), + |config| config.allow_origins(), + ); + } + + #[test] + fn list_allow_type() { + test_cors_config( + "allow_methods = [\"GET\"]", + &AllowType::List(vec![Method::GET]), + |config| config.allow_methods(), + ); + } + + #[test] + fn tagged_any_allow_type() { + test_cors_config( + "expose_headers = \"All\"", + &AllowType::Tagged(TaggedAllowTypes::All), + |config| config.expose_headers(), + ); + } + + #[test] + fn tagged_any_allow_type_err_on_mirror() { + let allow_type_method = "expose_headers = \"Mirror\""; + let config: Result = toml::from_str(allow_type_method); + assert!(config.is_err()); + } +} diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs new file mode 100644 index 00000000..e7affc2f --- /dev/null +++ b/htsget-config/src/config/advanced/mod.rs @@ -0,0 +1,4 @@ +//! Advanced configuration. +//! + +pub mod cors; diff --git a/htsget-config/src/config/data_server.rs b/htsget-config/src/config/data_server.rs index f6f71041..5d99bfff 100644 --- a/htsget-config/src/config/data_server.rs +++ b/htsget-config/src/config/data_server.rs @@ -1,6 +1,7 @@ //! Data server configuration. //! +use crate::config::advanced::cors::CorsConfig; use crate::config::{default_localstorage_addr, default_path}; use crate::tls::TlsServerConfig; use serde::{Deserialize, Serialize}; @@ -15,15 +16,22 @@ pub struct DataServerConfig { local_path: PathBuf, #[serde(skip_serializing)] tls: Option, + cors: CorsConfig, } impl DataServerConfig { /// Create the ticket server config. - pub fn new(addr: SocketAddr, local_path: PathBuf, tls: Option) -> Self { + pub fn new( + addr: SocketAddr, + local_path: PathBuf, + tls: Option, + cors: CorsConfig, + ) -> Self { Self { addr, local_path, tls, + cors, } } @@ -41,6 +49,11 @@ impl DataServerConfig { pub fn tls(&self) -> Option<&TlsServerConfig> { self.tls.as_ref() } + + /// Get the CORS config. + pub fn cors(&self) -> &CorsConfig { + &self.cors + } } impl Default for DataServerConfig { @@ -50,7 +63,8 @@ impl Default for DataServerConfig { .parse() .expect("expected valid address"), local_path: default_path().into(), - tls: None, + tls: Default::default(), + cors: Default::default(), } } } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 0a0d8add..95f9f8ee 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -22,9 +22,10 @@ use crate::error::Result; use crate::resolver::Resolver; use crate::tls::TlsServerConfig; +pub mod advanced; pub mod cors; pub mod data_server; -mod location; +pub mod location; pub mod parser; pub mod service_info; pub mod simple; diff --git a/htsget-config/src/config/ticket_server.rs b/htsget-config/src/config/ticket_server.rs index ce6db032..515eccec 100644 --- a/htsget-config/src/config/ticket_server.rs +++ b/htsget-config/src/config/ticket_server.rs @@ -1,6 +1,7 @@ //! Ticket server configuration. //! +use crate::config::advanced::cors::CorsConfig; use crate::config::default_addr; use crate::tls::TlsServerConfig; use serde::{Deserialize, Serialize}; @@ -13,12 +14,13 @@ pub struct TicketServerConfig { addr: SocketAddr, #[serde(skip_serializing)] tls: Option, + cors: CorsConfig, } impl TicketServerConfig { /// Create the ticket server config. - pub fn new(addr: SocketAddr, tls: Option) -> Self { - Self { addr, tls } + pub fn new(addr: SocketAddr, tls: Option, cors: CorsConfig) -> Self { + Self { addr, tls, cors } } /// Get the socket address. @@ -30,13 +32,19 @@ impl TicketServerConfig { pub fn tls(&self) -> Option<&TlsServerConfig> { self.tls.as_ref() } + + /// Get the CORS config. + pub fn cors(&self) -> &CorsConfig { + &self.cors + } } impl Default for TicketServerConfig { fn default() -> Self { Self { addr: default_addr().parse().expect("expected valid address"), - tls: None, + tls: Default::default(), + cors: Default::default(), } } } From 9ac8052fa89592fe1b9efb053f1b5803846ba665 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Dec 2024 10:43:03 +1100 Subject: [PATCH 03/12] feat(config): split location into simple and regex-based --- htsget-config/src/config/advanced/mod.rs | 1 + .../src/config/advanced/regex_location.rs | 43 +++++++++++++++++++ htsget-config/src/config/location.rs | 15 +++++++ htsget-config/src/config/simple.rs | 10 ++--- 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 htsget-config/src/config/advanced/regex_location.rs diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index e7affc2f..480560e1 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -2,3 +2,4 @@ //! pub mod cors; +pub mod regex_location; diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs new file mode 100644 index 00000000..bf8601db --- /dev/null +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -0,0 +1,43 @@ +//! Set the location using a regex and substitution values. +//! + +use regex::Regex; +use serde::{Deserialize, Serialize}; + +/// A regex storage is a storage that matches ids using Regex. +#[derive(Serialize, Debug, Clone, Deserialize)] +#[serde(default)] +pub struct RegexLocation { + #[serde(with = "serde_regex")] + regex: Regex, + substitution_string: String, +} + +impl RegexLocation { + /// Create a new regex location. + pub fn new(regex: Regex, substitution_string: String) -> Self { + Self { + regex, + substitution_string, + } + } + + /// Get the regex. + pub fn regex(&self) -> &Regex { + &self.regex + } + + /// Get the substitution string. + pub fn substitution_string(&self) -> &str { + &self.substitution_string + } +} + +impl Default for RegexLocation { + fn default() -> Self { + Self::new( + ".*".parse().expect("expected valid regex"), + "$0".to_string(), + ) + } +} diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index c0437e05..dcaaf72c 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -1,10 +1,25 @@ //! Location configuration. //! +use crate::config::advanced::regex_location::RegexLocation; use crate::config::location::Endpoint::{File, Url, S3}; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; +/// Either simple or regex based location +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum LocationEither { + Simple(Location), + Regex(RegexLocation), +} + +impl Default for LocationEither { + fn default() -> Self { + Self::Simple(Default::default()) + } +} + /// Location config. #[derive(Serialize, Debug, Clone, Default)] #[serde(default)] diff --git a/htsget-config/src/config/simple.rs b/htsget-config/src/config/simple.rs index 5fb95e01..9856c7ed 100644 --- a/htsget-config/src/config/simple.rs +++ b/htsget-config/src/config/simple.rs @@ -2,7 +2,7 @@ //! use crate::config::data_server::DataServerConfig; -use crate::config::location::Location; +use crate::config::location::LocationEither; use crate::config::service_info::ServiceInfo; use crate::config::ticket_server::TicketServerConfig; use serde::{Deserialize, Serialize}; @@ -14,7 +14,7 @@ pub struct Config { ticket_server: TicketServerConfig, data_server: Option, service_info: Option, - location: Option, + location: LocationEither, } impl Config { @@ -23,7 +23,7 @@ impl Config { ticket_server: TicketServerConfig, data_server: Option, service_info: Option, - location: Option, + location: LocationEither, ) -> Self { Self { ticket_server, @@ -49,7 +49,7 @@ impl Config { } /// Get the location. - pub fn location(&self) -> Option<&Location> { - self.location.as_ref() + pub fn location(&self) -> &LocationEither { + &self.location } } From 6e7fad90c5f7f66ee5665d07143a65d661e86041 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Dec 2024 13:24:05 +1100 Subject: [PATCH 04/12] feat(config): add complex storage options to advanced configuration --- htsget-config/src/config/advanced/file.rs | 77 +++++++++++++++++++ htsget-config/src/config/advanced/mod.rs | 5 ++ .../src/config/advanced/regex_location.rs | 36 ++++++++- htsget-config/src/config/advanced/s3.rs | 54 +++++++++++++ htsget-config/src/config/advanced/url.rs | 75 ++++++++++++++++++ htsget-config/src/config/location.rs | 16 ++-- 6 files changed, 254 insertions(+), 9 deletions(-) create mode 100644 htsget-config/src/config/advanced/file.rs create mode 100644 htsget-config/src/config/advanced/s3.rs create mode 100644 htsget-config/src/config/advanced/url.rs diff --git a/htsget-config/src/config/advanced/file.rs b/htsget-config/src/config/advanced/file.rs new file mode 100644 index 00000000..8982841f --- /dev/null +++ b/htsget-config/src/config/advanced/file.rs @@ -0,0 +1,77 @@ +//! Configuration of local file based storage. +//! + +use crate::config::{default_localstorage_addr, default_path}; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; +use crate::types::Scheme; +use http::uri::Authority; +use serde::{Deserialize, Serialize}; + +/// Local file based storage. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct File { + scheme: Scheme, + #[serde(with = "http_serde::authority")] + authority: Authority, + local_path: String, + #[serde(skip_serializing)] + #[cfg(feature = "experimental")] + keys: Option, +} + +impl File { + /// Create a new local storage. + pub fn new(scheme: Scheme, authority: Authority, local_path: String) -> Self { + Self { + scheme, + authority, + local_path, + #[cfg(feature = "experimental")] + keys: None, + } + } + + /// Get the scheme. + pub fn scheme(&self) -> Scheme { + self.scheme + } + + /// Get the authority. + pub fn authority(&self) -> &Authority { + &self.authority + } + + /// Get the local path. + pub fn local_path(&self) -> &str { + &self.local_path + } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } +} + +impl Default for File { + fn default() -> Self { + Self::new(Scheme::Http, default_authority(), default_local_path()) + } +} + +fn default_authority() -> Authority { + Authority::from_static(default_localstorage_addr()) +} + +fn default_local_path() -> String { + default_path().into() +} diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index 480560e1..d81465a6 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -2,4 +2,9 @@ //! pub mod cors; +pub mod file; pub mod regex_location; +#[cfg(feature = "s3-storage")] +pub mod s3; +#[cfg(feature = "url-storage")] +pub mod url; diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index bf8601db..ffb7b7a9 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -1,9 +1,35 @@ //! Set the location using a regex and substitution values. //! +use crate::config::advanced::file::File; +#[cfg(feature = "s3-storage")] +use crate::config::advanced::s3::S3; +#[cfg(feature = "url-storage")] +use crate::config::advanced::url::UrlStorage; use regex::Regex; use serde::{Deserialize, Serialize}; +/// Specify the storage backend to use as config values. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "backend")] +#[non_exhaustive] +pub enum Backend { + #[serde(alias = "local", alias = "LOCAL")] + Local(File), + #[cfg(feature = "s3-storage")] + #[serde(alias = "s3")] + S3(S3), + #[cfg(feature = "url-storage")] + #[serde(alias = "url", alias = "URL")] + Url(UrlStorage), +} + +impl Default for Backend { + fn default() -> Self { + Self::Local(Default::default()) + } +} + /// A regex storage is a storage that matches ids using Regex. #[derive(Serialize, Debug, Clone, Deserialize)] #[serde(default)] @@ -11,14 +37,16 @@ pub struct RegexLocation { #[serde(with = "serde_regex")] regex: Regex, substitution_string: String, + storage: Backend, } impl RegexLocation { /// Create a new regex location. - pub fn new(regex: Regex, substitution_string: String) -> Self { + pub fn new(regex: Regex, substitution_string: String, storage: Backend) -> Self { Self { regex, substitution_string, + storage, } } @@ -31,6 +59,11 @@ impl RegexLocation { pub fn substitution_string(&self) -> &str { &self.substitution_string } + + /// Get the storage backend. + pub fn backend(&self) -> &Backend { + &self.storage + } } impl Default for RegexLocation { @@ -38,6 +71,7 @@ impl Default for RegexLocation { Self::new( ".*".parse().expect("expected valid regex"), "$0".to_string(), + Default::default(), ) } } diff --git a/htsget-config/src/config/advanced/s3.rs b/htsget-config/src/config/advanced/s3.rs new file mode 100644 index 00000000..abc051ba --- /dev/null +++ b/htsget-config/src/config/advanced/s3.rs @@ -0,0 +1,54 @@ +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct S3 { + bucket: String, + endpoint: Option, + path_style: bool, + #[serde(skip_serializing)] + #[cfg(feature = "experimental")] + keys: Option, +} + +impl S3 { + /// Create a new S3 storage. + pub fn new(bucket: String, endpoint: Option, path_style: bool) -> Self { + Self { + bucket, + endpoint, + path_style, + #[cfg(feature = "experimental")] + keys: None, + } + } + + /// Get the bucket. + pub fn bucket(&self) -> &str { + &self.bucket + } + + /// Get the endpoint + pub fn endpoint(&self) -> Option<&str> { + self.endpoint.as_deref() + } + + /// Get the path style + pub fn path_style(&self) -> bool { + self.path_style + } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } +} diff --git a/htsget-config/src/config/advanced/url.rs b/htsget-config/src/config/advanced/url.rs new file mode 100644 index 00000000..48544aeb --- /dev/null +++ b/htsget-config/src/config/advanced/url.rs @@ -0,0 +1,75 @@ +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; +use crate::tls::client::TlsClientConfig; +use http::Uri; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct UrlStorage { + #[serde(with = "http_serde::uri")] + uri: Uri, + #[serde(with = "http_serde::uri")] + response_uri: Uri, + forward_headers: bool, + header_blacklist: Vec, + #[serde(skip_serializing)] + tls: TlsClientConfig, + #[serde(skip_serializing)] + #[cfg(feature = "experimental")] + keys: Option, +} + +impl UrlStorage { + /// Create a new url storage. + pub fn new( + uri: Uri, + response_uri: Uri, + forward_headers: bool, + header_blacklist: Vec, + tls: TlsClientConfig, + ) -> Self { + Self { + uri, + response_uri, + forward_headers, + header_blacklist, + tls, + #[cfg(feature = "experimental")] + keys: None, + } + } + + /// Get the uri called when resolving the query. + pub fn uri(&self) -> &Uri { + &self.uri + } + + /// Get the response uri which is returned to the client. + pub fn response_uri(&self) -> &Uri { + &self.response_uri + } + + /// Whether headers received in a query request should be + /// included in the returned data block tickets. + pub fn forward_headers(&self) -> bool { + self.forward_headers + } + + /// Get the tls client config. + pub fn tls(&self) -> &TlsClientConfig { + &self.tls + } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } +} diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index dcaaf72c..39443fc2 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -2,7 +2,7 @@ //! use crate::config::advanced::regex_location::RegexLocation; -use crate::config::location::Endpoint::{File, Url, S3}; +use crate::config::location::Backend::{File, Url, S3}; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; @@ -24,19 +24,19 @@ impl Default for LocationEither { #[derive(Serialize, Debug, Clone, Default)] #[serde(default)] pub struct Location { - endpoint: Endpoint, + backend: Backend, prefix: String, } impl Location { /// Create a new location. - pub fn new(endpoint: Endpoint, prefix: String) -> Self { - Self { endpoint, prefix } + pub fn new(backend: Backend, prefix: String) -> Self { + Self { backend, prefix } } - /// Get the endpoint. - pub fn endpoint(&self) -> Endpoint { - self.endpoint + /// Get the storage backend. + pub fn backend(&self) -> Backend { + self.backend } /// Get the prefix. @@ -46,7 +46,7 @@ impl Location { } #[derive(Serialize, Deserialize, Debug, Clone, Default, Copy)] -pub enum Endpoint { +pub enum Backend { #[default] File, S3, From 427be44776a62b31c5be037217594cee12697a82 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Dec 2024 13:57:44 +1100 Subject: [PATCH 05/12] feat(config): add allow guard to advanced config --- .../src/config/advanced/allow_guard.rs | 76 +++++++++++++++++++ htsget-config/src/config/advanced/mod.rs | 1 + .../src/config/advanced/regex_location.rs | 17 ++++- 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 htsget-config/src/config/advanced/allow_guard.rs diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs new file mode 100644 index 00000000..6d15c5fa --- /dev/null +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -0,0 +1,76 @@ +//! Allow guard configuration. +//! + +use crate::types::{Class, Fields, Format, Interval, TaggedTypeAll, Tags}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +/// A query guard represents query parameters that can be allowed to storage for a given query. +#[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct AllowGuard { + allow_reference_names: ReferenceNames, + allow_fields: Fields, + allow_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + allow_interval: Interval, +} + +/// Reference names that can be matched. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum ReferenceNames { + Tagged(TaggedTypeAll), + List(HashSet), +} + +impl AllowGuard { + /// Create a new allow guard. + pub fn new( + allow_reference_names: ReferenceNames, + allow_fields: Fields, + allow_tags: Tags, + allow_formats: Vec, + allow_classes: Vec, + allow_interval: Interval, + ) -> Self { + Self { + allow_reference_names, + allow_fields, + allow_tags, + allow_formats, + allow_classes, + allow_interval, + } + } + + /// Get allow formats. + pub fn allow_formats(&self) -> &[Format] { + &self.allow_formats + } + + /// Get allow classes. + pub fn allow_classes(&self) -> &[Class] { + &self.allow_classes + } + + /// Get allow interval. + pub fn allow_interval(&self) -> Interval { + self.allow_interval + } + + /// Get allow reference names. + pub fn allow_reference_names(&self) -> &ReferenceNames { + &self.allow_reference_names + } + + /// Get allow fields. + pub fn allow_fields(&self) -> &Fields { + &self.allow_fields + } + + /// Get allow tags. + pub fn allow_tags(&self) -> &Tags { + &self.allow_tags + } +} diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index d81465a6..58d57629 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -1,6 +1,7 @@ //! Advanced configuration. //! +pub mod allow_guard; pub mod cors; pub mod file; pub mod regex_location; diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index ffb7b7a9..98c9f0d8 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -1,6 +1,7 @@ //! Set the location using a regex and substitution values. //! +use crate::config::advanced::allow_guard::AllowGuard; use crate::config::advanced::file::File; #[cfg(feature = "s3-storage")] use crate::config::advanced::s3::S3; @@ -37,16 +38,23 @@ pub struct RegexLocation { #[serde(with = "serde_regex")] regex: Regex, substitution_string: String, - storage: Backend, + backend: Backend, + guard: Option, } impl RegexLocation { /// Create a new regex location. - pub fn new(regex: Regex, substitution_string: String, storage: Backend) -> Self { + pub fn new( + regex: Regex, + substitution_string: String, + backend: Backend, + guard: Option, + ) -> Self { Self { regex, substitution_string, - storage, + backend, + guard, } } @@ -62,7 +70,7 @@ impl RegexLocation { /// Get the storage backend. pub fn backend(&self) -> &Backend { - &self.storage + &self.backend } } @@ -72,6 +80,7 @@ impl Default for RegexLocation { ".*".parse().expect("expected valid regex"), "$0".to_string(), Default::default(), + Default::default(), ) } } From 2e17c7d7c0bcb40c39bd3747b0d2c1e9406eff4a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 13 Dec 2024 11:21:58 +1100 Subject: [PATCH 06/12] feat: refactor config for new simplified location --- .pre-commit-config.yaml | 2 - Cargo.lock | 1248 ++++++++++------- htsget-actix/benches/request_benchmarks.rs | 8 +- htsget-actix/src/handlers/service_info.rs | 2 +- htsget-actix/src/lib.rs | 20 +- htsget-actix/src/main.rs | 9 +- htsget-axum/src/handlers/service_info.rs | 2 +- htsget-axum/src/main.rs | 5 +- htsget-axum/src/server/data.rs | 44 +- htsget-axum/src/server/mod.rs | 25 +- htsget-axum/src/server/ticket.rs | 21 +- .../src/config/advanced/allow_guard.rs | 228 ++- htsget-config/src/config/advanced/cors.rs | 67 +- htsget-config/src/config/advanced/mod.rs | 15 +- .../src/config/advanced/regex_location.rs | 122 +- htsget-config/src/config/advanced/s3.rs | 54 - htsget-config/src/config/advanced/url.rs | 117 +- htsget-config/src/config/cors.rs | 298 ---- htsget-config/src/config/data_server.rs | 59 +- htsget-config/src/config/location.rs | 365 ++++- htsget-config/src/config/mod.rs | 586 ++------ htsget-config/src/config/parser.rs | 21 +- htsget-config/src/config/service_info.rs | 39 +- htsget-config/src/config/simple.rs | 55 - htsget-config/src/config/ticket_server.rs | 29 +- htsget-config/src/resolver.rs | 704 ++++------ htsget-config/src/storage/c4gh/local.rs | 26 +- htsget-config/src/storage/c4gh/mod.rs | 18 +- .../src/storage/c4gh/secrets_manager.rs | 1 + .../src/{config/advanced => storage}/file.rs | 66 +- htsget-config/src/storage/local.rs | 180 --- htsget-config/src/storage/mod.rs | 96 +- htsget-config/src/storage/s3.rs | 66 +- htsget-config/src/storage/url.rs | 265 +--- htsget-config/src/tls/mod.rs | 6 +- htsget-config/src/types.rs | 9 +- htsget-http/Cargo.toml | 1 + htsget-http/src/lib.rs | 21 +- htsget-http/src/service_info.rs | 163 +-- htsget-lambda/src/main.rs | 2 +- htsget-search/benches/search_benchmarks.rs | 8 +- htsget-search/src/from_storage.rs | 74 +- htsget-search/src/lib.rs | 9 +- htsget-storage/src/lib.rs | 75 +- htsget-storage/src/local.rs | 23 +- htsget-test/src/http/mod.rs | 70 +- htsget-test/src/http/server.rs | 38 +- htsget-test/src/lib.rs | 6 - htsget-test/src/util.rs | 2 +- 49 files changed, 2557 insertions(+), 2813 deletions(-) delete mode 100644 htsget-config/src/config/advanced/s3.rs delete mode 100644 htsget-config/src/config/cors.rs delete mode 100644 htsget-config/src/config/simple.rs rename htsget-config/src/{config/advanced => storage}/file.rs (50%) delete mode 100644 htsget-config/src/storage/local.rs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f36d3ae5..9cec04b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,5 +25,3 @@ repos: rev: v1.0 hooks: - id: fmt - - id: clippy - args: ["--all-targets", "--all-features", "--", "-D", "warnings"] diff --git a/Cargo.lock b/Cargo.lock index 5e7ecfc7..9e90172d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -151,7 +151,7 @@ dependencies = [ "pin-project-lite", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tracing", ] @@ -218,14 +218,14 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -323,9 +323,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -338,36 +338,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "flate2", "futures-core", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -413,13 +413,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -430,7 +430,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -465,9 +465,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.7" +version = "1.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" dependencies = [ "aws-credential-types", "aws-runtime", @@ -476,7 +476,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -507,21 +507,20 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.9.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3" dependencies = [ "aws-lc-sys", - "mirai-annotations", "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.21.2" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62" +checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7" dependencies = [ "bindgen", "cc", @@ -534,9 +533,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -560,11 +559,10 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.52.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f571deb0a80c20d21d9f3e8418c1712af9ff4bf399d057e5549a934eca4844e2" +checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" dependencies = [ - "ahash", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -572,7 +570,7 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -595,15 +593,15 @@ dependencies = [ [[package]] name = "aws-sdk-secretsmanager" -version = "1.49.0" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2ac19e43e100834e7b9e6f838af7506a5cb8ee7531d1104cb207b3d927e0c2" +checksum = "450b2e8cb5f0ee102e4a04c5f8e923aff8187ae9323058707c6cec238cf51699" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -618,15 +616,15 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.44.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -640,15 +638,15 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.45.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -662,15 +660,15 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.44.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -685,9 +683,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -700,7 +698,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "once_cell", "p256", "percent-encoding", @@ -725,9 +723,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.12" +version = "0.60.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -785,6 +783,15 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + [[package]] name = "aws-smithy-mocks-experimental" version = "0.2.1" @@ -797,21 +804,21 @@ dependencies = [ [[package]] name = "aws-smithy-protocol-test" -version = "0.62.0" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495c940cd5c7232ac3f0945ff559096deadd2fc73e4418a0e98fe5836788bb39" +checksum = "3b92b62199921f10685c6b588fdbeb81168ae4e7950ae3e5f50145a01bb5f1ad" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", "base64-simd", "cbor-diag", + "ciborium", "http 0.2.12", "pretty_assertions", "regex-lite", "roxmltree", - "serde_cbor", "serde_json", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -826,9 +833,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -842,9 +849,9 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "hyper 0.14.30", + "hyper 0.14.31", "hyper-rustls 0.24.2", - "indexmap 2.5.0", + "indexmap 2.7.0", "once_cell", "pin-project-lite", "pin-utils", @@ -858,15 +865,15 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "tokio", "tracing", @@ -875,16 +882,16 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -940,7 +947,7 @@ checksum = "7319a086b79c3ff026a33a61e80f04fd3885fbb73237981ea080d21944e1cb1c" dependencies = [ "base64 0.22.1", "bytes", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-serde", "query_map", @@ -950,18 +957,18 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.7" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.1", "hyper-util", "itoa", "matchit", @@ -974,9 +981,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -991,13 +998,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -1005,25 +1012,27 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.4" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c3220b188aea709cf1b6c5f9b01c3bd936bb08bd2b5184a12b35ac8131b1f9" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" dependencies = [ "axum", "axum-core", "bytes", + "fastrand", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", + "multer", "pin-project-lite", "serde", "serde_json", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", - "tracing", + "typed-json", ] [[package]] @@ -1097,9 +1106,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags", "cexpr", @@ -1114,7 +1123,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.79", + "syn 2.0.90", "which", ] @@ -1199,9 +1208,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" dependencies = [ "memchr", "serde", @@ -1215,9 +1224,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -1227,9 +1236,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" dependencies = [ "serde", ] @@ -1246,9 +1255,9 @@ dependencies = [ [[package]] name = "bytestring" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f" dependencies = [ "bytes", ] @@ -1298,7 +1307,7 @@ dependencies = [ "bs58", "chrono", "data-encoding", - "half 2.4.1", + "half", "nom", "num-bigint", "num-rational", @@ -1310,9 +1319,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.22" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -1334,6 +1343,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -1360,9 +1375,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1395,7 +1410,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 2.4.1", + "half", ] [[package]] @@ -1422,9 +1437,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -1432,9 +1447,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -1451,29 +1466,29 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" dependencies = [ "cc", ] [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "const-oid" @@ -1516,9 +1531,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -1645,7 +1660,7 @@ dependencies = [ "rpassword", "scrypt", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1724,7 +1739,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1748,7 +1763,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1759,7 +1774,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1804,7 +1819,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -1824,6 +1839,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1876,9 +1902,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -1904,12 +1930,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1926,9 +1952,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -1964,9 +1990,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -1978,6 +2004,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1995,9 +2027,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -2010,9 +2042,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -2020,15 +2052,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -2037,38 +2069,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -2099,15 +2131,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -2138,7 +2172,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2147,29 +2181,23 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.1.0", - "indexmap 2.5.0", + "http 1.2.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - [[package]] name = "half" version = "2.4.1" @@ -2188,12 +2216,13 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "ahash", "allocator-api2", + "equivalent", + "foldhash", ] [[package]] @@ -2202,12 +2231,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hermit-abi" version = "0.4.0" @@ -2264,10 +2287,10 @@ dependencies = [ "htsget-search", "htsget-test", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "reqwest", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.20", + "rustls-pemfile 2.2.0", "serde", "serde_json", "tempfile", @@ -2289,16 +2312,16 @@ dependencies = [ "htsget-http", "htsget-search", "htsget-test", - "http 1.1.0", - "hyper 1.4.1", + "http 1.2.0", + "hyper 1.5.1", "hyper-util", "reqwest", - "rustls 0.23.13", + "rustls 0.23.20", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.0", - "tower 0.5.1", + "tokio-rustls 0.26.1", + "tower 0.5.2", "tower-http", "tracing", ] @@ -2316,21 +2339,21 @@ dependencies = [ "crypt4gh", "figment", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-serde", "noodles", "rcgen", "regex", "reqwest", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.20", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_regex", "serde_with", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "toml", "tracing", @@ -2345,9 +2368,10 @@ dependencies = [ "htsget-config", "htsget-search", "htsget-test", - "http 1.1.0", + "http 1.2.0", "serde", - "thiserror", + "serde_json", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -2368,7 +2392,7 @@ dependencies = [ "mime", "query_map", "regex", - "rustls 0.23.13", + "rustls 0.23.20", "serde", "serde_json", "tempfile", @@ -2389,10 +2413,10 @@ dependencies = [ "htsget-config", "htsget-storage", "htsget-test", - "http 1.1.0", + "http 1.2.0", "noodles", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -2416,11 +2440,11 @@ dependencies = [ "futures-util", "htsget-config", "htsget-test", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "reqwest", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-util", "tower-http", @@ -2440,7 +2464,7 @@ dependencies = [ "crypt4gh", "futures", "htsget-config", - "http 1.1.0", + "http 1.2.0", "mime", "noodles", "rcgen", @@ -2451,7 +2475,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", ] @@ -2468,9 +2492,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -2495,7 +2519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -2506,16 +2530,16 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "http-range-header" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" [[package]] name = "http-serde" @@ -2523,15 +2547,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http 1.1.0", + "http 1.2.0", "serde", ] [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -2547,9 +2571,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" dependencies = [ "bytes", "futures-channel", @@ -2571,15 +2595,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", - "http 1.1.0", + "h2 0.4.7", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", @@ -2598,7 +2622,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.30", + "hyper 0.14.31", "log", "rustls 0.21.12", "rustls-native-certs", @@ -2613,29 +2637,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.4.1", + "http 1.2.0", + "hyper 1.5.1", "hyper-util", - "rustls 0.23.13", + "rustls 0.23.20", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tower-service", "webpki-roots", ] [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.1", "pin-project-lite", "socket2", "tokio", @@ -2666,6 +2690,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2674,19 +2816,30 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] name = "impl-more" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" [[package]] name = "indexmap" @@ -2701,12 +2854,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "serde", ] @@ -2728,9 +2881,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-terminal" @@ -2738,7 +2891,7 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi 0.4.0", + "hermit-abi", "libc", "windows-sys 0.52.0", ] @@ -2778,9 +2931,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -2793,10 +2946,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2812,10 +2966,10 @@ dependencies = [ "encoding_rs", "futures", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.1", "lambda_runtime", "mime", "percent-encoding", @@ -2837,11 +2991,11 @@ dependencies = [ "base64 0.22.1", "bytes", "futures", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "http-serde", - "hyper 1.4.1", + "hyper 1.5.1", "hyper-util", "lambda_runtime_api_client", "pin-project", @@ -2864,10 +3018,10 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.1", "hyper-util", "tokio", "tower 0.4.13", @@ -2896,9 +3050,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lexical-core" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2909,9 +3063,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -2920,9 +3074,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ "lexical-util", "static_assertions", @@ -2930,18 +3084,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" dependencies = [ "lexical-util", "lexical-write-integer", @@ -2950,9 +3104,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" dependencies = [ "lexical-util", "static_assertions", @@ -2960,15 +3114,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.159" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -2980,6 +3134,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + [[package]] name = "local-channel" version = "0.1.5" @@ -3015,11 +3175,11 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] @@ -3097,11 +3257,10 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi 0.3.9", "libc", "log", "wasi", @@ -3109,10 +3268,21 @@ dependencies = [ ] [[package]] -name = "mirai-annotations" -version = "1.12.0" +name = "multer" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.2.0", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] [[package]] name = "mutually_exclusive_features" @@ -3160,7 +3330,7 @@ dependencies = [ "byteorder", "bytes", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3176,7 +3346,7 @@ checksum = "e465d9d332ee4f5366cb1861aae94177767504f4a490277ed33a16ffb372b47e" dependencies = [ "byteorder", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3223,7 +3393,7 @@ dependencies = [ "bzip2", "flate2", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "md-5", "noodles-bam", "noodles-core", @@ -3242,7 +3412,7 @@ checksum = "a0f41004636fb4232155421cbf4706565073623838a8252875085fa670b8185c" dependencies = [ "bit-vec", "byteorder", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "tokio", @@ -3281,7 +3451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca42cb034ccc595d963de2ff2fd9cf2c3f563bcb65c1337e90f9d73724743d84" dependencies = [ "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3298,7 +3468,7 @@ dependencies = [ "bitflags", "bstr", "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "lexical-core", "memchr", "noodles-bgzf", @@ -3314,7 +3484,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5c5ed1fa0b9ae083c2e1e1cedc07715861400fdd943fdb1ed6c593719be187" dependencies = [ "byteorder", - "indexmap 2.5.0", + "indexmap 2.7.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3328,7 +3498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be568315676d38294c4b00bd608bd19d04394e6106c2db08c4b351b275741df" dependencies = [ "futures", - "indexmap 2.5.0", + "indexmap 2.7.0", "memchr", "noodles-bgzf", "noodles-core", @@ -3411,21 +3581,18 @@ checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -3556,7 +3723,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -3577,29 +3744,29 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -3662,12 +3829,6 @@ dependencies = [ "universal-hash", ] -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - [[package]] name = "powerfmt" version = "0.2.0" @@ -3705,12 +3866,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -3739,9 +3900,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -3754,7 +3915,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", "version_check", "yansi", ] @@ -3782,45 +3943,49 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustc-hash 2.1.0", + "rustls 0.23.20", "socket2", - "thiserror", + "thiserror 2.0.6", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", - "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustc-hash 2.1.0", + "rustls 0.23.20", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.6", "tinyvec", "tracing", + "web-time", ] [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -3902,22 +4067,22 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -3932,9 +4097,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -3961,19 +4126,19 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.1", "hyper-rustls 0.27.3", "hyper-util", "ipnet", @@ -3984,15 +4149,15 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.20", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tower-service", "url", @@ -4074,9 +4239,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc_version" @@ -4089,15 +4254,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4114,9 +4279,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "aws-lc-rs", "log", @@ -4151,19 +4316,21 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -4189,9 +4356,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -4221,7 +4388,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "httparse", - "hyper 1.4.1", + "hyper 1.5.1", "itoa", "memchr", "mime", @@ -4235,8 +4402,8 @@ dependencies = [ "sha1", "sha2", "smallvec", - "sync_wrapper 1.0.1", - "thiserror", + "sync_wrapper", + "thiserror 1.0.69", "time", "tokio", "tracing", @@ -4256,9 +4423,9 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-types-convert", - "hyper 1.4.1", + "hyper 1.5.1", "s3s", - "sync_wrapper 1.0.1", + "sync_wrapper", "tracing", "transform-stream", ] @@ -4283,7 +4450,7 @@ dependencies = [ "path-absolutize", "s3s", "serde_json", - "thiserror", + "thiserror 1.0.69", "time", "tokio", "tokio-util", @@ -4313,9 +4480,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -4377,9 +4544,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -4387,9 +4554,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "separator" @@ -4399,41 +4566,31 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half 1.8.3", - "serde", -] - [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.7.0", "itoa", "memchr", "ryu", @@ -4483,15 +4640,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.7.0", "serde", "serde_derive", "serde_json", @@ -4501,14 +4658,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -4590,9 +4747,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4614,6 +4771,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -4645,9 +4808,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -4656,24 +4819,29 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] -name = "sync_wrapper" -version = "1.0.1" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ - "futures-core", + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -4693,22 +4861,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.6", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -4723,9 +4911,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -4744,14 +4932,24 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4779,9 +4977,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -4803,7 +5001,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] @@ -4818,20 +5016,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.13", - "rustls-pki-types", + "rustls 0.23.20", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -4840,9 +5037,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -4879,7 +5076,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -4903,14 +5100,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -4919,14 +5116,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "http-range-header", @@ -4956,9 +5153,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -4968,9 +5165,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15bc0cd5f72e837e310f4d978a90abf202a7f7d8ef3272246bae381d0086d3bf" +checksum = "54a9f5c1aca50ebebf074ee665b9f99f2e84906dcf6b993a0d0090edb835166d" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -4981,20 +5178,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -5002,9 +5199,9 @@ dependencies = [ [[package]] name = "tracing-error" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" dependencies = [ "tracing", "tracing-subscriber", @@ -5023,9 +5220,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ "serde", "tracing-core", @@ -5033,9 +5230,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -5067,6 +5264,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6024a8d0025400b3f6b189366e9aa92012cf9c4fe1cd2620848dd61425c49eed" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "typenum" version = "1.17.0" @@ -5084,33 +5291,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.15" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" - -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "universal-hash" @@ -5130,9 +5319,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -5145,6 +5334,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -5153,9 +5354,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", ] @@ -5205,9 +5406,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -5216,36 +5417,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5253,28 +5454,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -5285,9 +5486,19 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -5295,9 +5506,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -5541,6 +5752,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xmlparser" version = "0.13.6" @@ -5571,6 +5794,30 @@ dependencies = [ "time", ] +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -5589,7 +5836,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.90", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", + "synstructure", ] [[package]] @@ -5598,6 +5866,28 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/htsget-actix/benches/request_benchmarks.rs b/htsget-actix/benches/request_benchmarks.rs index 1dd6b5d9..d13184b2 100644 --- a/htsget-actix/benches/request_benchmarks.rs +++ b/htsget-actix/benches/request_benchmarks.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use htsget_config::types::{Headers, JsonResponse}; use htsget_http::{PostRequest, Region}; use htsget_test::http::default_config_fixed_port; -use htsget_test::util::{default_dir, default_dir_data}; +use htsget_test::util::default_dir; const REFSERVER_DOCKER_IMAGE: &str = "ga4gh/htsget-refserver:1.5.0"; const BENCHMARK_DURATION_SECONDS: u64 = 30; @@ -144,14 +144,14 @@ fn start_htsget_rs() -> (DropGuard, String) { .arg("-p") .arg("htsget-actix") .arg("--no-default-features") - .env("HTSGET_PATH", default_dir_data()) - .env("RUST_LOG", "warn") .spawn() .unwrap(); let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let htsget_rs_ticket_url = format!("http://{}", config.data_server().addr()); + + let data_server = config.data_server().clone().unwrap(); + let htsget_rs_ticket_url = format!("http://{}", data_server.addr()); query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); (DropGuard(child), htsget_rs_url) diff --git a/htsget-actix/src/handlers/service_info.rs b/htsget-actix/src/handlers/service_info.rs index 8df7c6dc..f69e2a6b 100644 --- a/htsget-actix/src/handlers/service_info.rs +++ b/htsget-actix/src/handlers/service_info.rs @@ -21,7 +21,7 @@ pub fn get_service_info_json( PrettyJson(get_base_service_info_json( endpoint, app_state.htsget.clone(), - &app_state.config_service_info, + app_state.config_service_info.clone(), )) } diff --git a/htsget-actix/src/lib.rs b/htsget-actix/src/lib.rs index 2d7efb2b..fd0a1991 100644 --- a/htsget-actix/src/lib.rs +++ b/htsget-actix/src/lib.rs @@ -5,9 +5,10 @@ use tracing::info; use tracing::instrument; use tracing_actix_web::TracingLogger; -use htsget_config::config::cors::CorsConfig; -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig, USAGE}; -pub use htsget_config::storage::Storage; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::service_info::ServiceInfo; +use htsget_config::config::ticket_server::TicketServerConfig; +pub use htsget_config::config::{Config, USAGE}; use htsget_search::HtsGet; use crate::handlers::{get, post, reads_service_info, variants_service_info, HttpVersionCompat}; @@ -214,14 +215,13 @@ mod tests { #[async_trait(?Send)] impl TestServer> for ActixTestServer { async fn get_expected_path(&self) -> String { - let mut bind_data_server = BindServer::from(self.get_config().data_server().clone()); - let server = bind_data_server - .bind_data_server("/data".to_string()) - .await - .unwrap(); + let data_server = self.get_config().data_server().clone().unwrap(); + + let path = data_server.local_path().to_path_buf(); + let mut bind_data_server = BindServer::from(data_server); + let server = bind_data_server.bind_data_server().await.unwrap(); let addr = server.local_addr(); - let path = self.get_config().data_server().local_path().to_path_buf(); tokio::spawn(async move { server.serve(path).await.unwrap() }); expected_url_path(self.get_config(), addr.unwrap()) @@ -278,7 +278,7 @@ mod tests { .configure(|service_config: &mut web::ServiceConfig| { configure_server( service_config, - self.config.clone().owned_resolvers(), + self.config.clone().into_locations(), self.config.service_info().clone(), ); }) diff --git a/htsget-actix/src/main.rs b/htsget-actix/src/main.rs index 7a904370..6695e662 100644 --- a/htsget-actix/src/main.rs +++ b/htsget-actix/src/main.rs @@ -7,6 +7,7 @@ use htsget_actix::run_server; use htsget_actix::Config; use htsget_axum::server::data; use htsget_config::command; +use htsget_config::config::data_server::DataServerEnabled; #[actix_web::main] async fn main() -> io::Result<()> { @@ -21,8 +22,8 @@ async fn main() -> io::Result<()> { debug!(config = ?config, "config parsed"); - if config.data_server().enabled() { - let local_server = data::join_handle(config.data_server().clone()).await?; + if let DataServerEnabled::Some(data_server) = config.data_server() { + let local_server = data::join_handle(data_server.clone()).await?; let ticket_server_config = config.ticket_server().clone(); let service_info = config.service_info().clone(); @@ -30,7 +31,7 @@ async fn main() -> io::Result<()> { select! { local_server = local_server => Ok(local_server??), actix_server = run_server( - config.owned_resolvers(), + config.into_locations(), ticket_server_config, service_info )? => actix_server @@ -39,7 +40,7 @@ async fn main() -> io::Result<()> { let ticket_server_config = config.ticket_server().clone(); let service_info = config.service_info().clone(); - run_server(config.owned_resolvers(), ticket_server_config, service_info)?.await + run_server(config.into_locations(), ticket_server_config, service_info)?.await } } else { Ok(()) diff --git a/htsget-axum/src/handlers/service_info.rs b/htsget-axum/src/handlers/service_info.rs index 737203ba..89dc7716 100644 --- a/htsget-axum/src/handlers/service_info.rs +++ b/htsget-axum/src/handlers/service_info.rs @@ -16,7 +16,7 @@ pub fn get_service_info_json( ErasedJson::pretty(get_base_service_info_json( endpoint, app_state.htsget, - &app_state.service_info, + app_state.service_info, )) } diff --git a/htsget-axum/src/main.rs b/htsget-axum/src/main.rs index 291d861f..bac6a3e9 100644 --- a/htsget-axum/src/main.rs +++ b/htsget-axum/src/main.rs @@ -5,6 +5,7 @@ use tracing::debug; use htsget_axum::server::{data, ticket}; use htsget_config::command; +use htsget_config::config::data_server::DataServerEnabled; use htsget_config::config::Config; #[tokio::main] @@ -22,8 +23,8 @@ async fn main() -> io::Result<()> { debug!(config = ?config, "config parsed"); - if config.data_server().enabled() { - let local_server = data::join_handle(config.data_server().clone()).await?; + if let DataServerEnabled::Some(data_server) = config.data_server() { + let local_server = data::join_handle(data_server.clone()).await?; let ticket_server = ticket::join_handle(config).await?; select! { diff --git a/htsget-axum/src/server/data.rs b/htsget-axum/src/server/data.rs index ea269ee5..15ec8e81 100644 --- a/htsget-axum/src/server/data.rs +++ b/htsget-axum/src/server/data.rs @@ -4,8 +4,9 @@ use crate::error::Result; use crate::server::{configure_cors, BindServer, Server}; use axum::Router; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::DataServerConfig; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::data_server::DataServerConfig; +use htsget_config::storage::file::PATH_PREFIX; use std::net::SocketAddr; use std::path::Path; use tokio::task::JoinHandle; @@ -17,32 +18,24 @@ use tracing::info; #[derive(Debug)] pub struct DataServer { server: Server, - serve_at: String, cors: CorsConfig, } impl DataServer { /// Create a new data server. - pub fn new(server: Server, serve_at: String, cors: CorsConfig) -> Self { - Self { - server, - serve_at, - cors, - } + pub fn new(server: Server, cors: CorsConfig) -> Self { + Self { server, cors } } /// Run the data server, using the provided path, key and certificate. pub async fn serve>(self, path: P) -> Result<()> { - self - .server - .serve(Self::router(self.cors, &self.serve_at, path)) - .await + self.server.serve(Self::router(self.cors, path)).await } /// Create the router for the data server. - pub fn router>(cors: CorsConfig, serve_at: &str, path: P) -> Router { + pub fn router>(cors: CorsConfig, path: P) -> Router { Router::new() - .nest_service(serve_at, ServeDir::new(path)) + .nest_service(PATH_PREFIX, ServeDir::new(path)) .layer(configure_cors(cors)) .layer(TraceLayer::new_for_http()) } @@ -69,11 +62,8 @@ impl From for BindServer { /// Spawn a task to run the data server. pub async fn join_handle(config: DataServerConfig) -> Result>> { - let serve_at = config.serve_at().to_string(); let local_path = config.local_path().to_path_buf(); - let data_server = BindServer::from(config.clone()) - .bind_data_server(serve_at) - .await?; + let data_server = BindServer::from(config.clone()).bind_data_server().await?; info!(address = ?data_server.local_addr()?, "data server address bound to"); @@ -213,8 +203,11 @@ mod tests { let _ = aws_lc_rs::default_provider().install_default(); let (_, base_path) = create_local_test_files().await; - let config = config_with_tls(base_path.path()).data_server().clone(); - let server_config = config.into_tls().unwrap(); + let data_server = config_with_tls(base_path.path()) + .data_server() + .clone() + .unwrap(); + let server_config = data_server.into_tls().unwrap(); test_server("https", Some(server_config), base_path.path().to_path_buf()).await; } @@ -267,8 +260,11 @@ mod tests { let _ = aws_lc_rs::default_provider().install_default(); let tmp_dir = tempdir().unwrap(); - let config = config_with_tls(tmp_dir.path()).data_server().clone(); - let server_config = config.into_tls().unwrap(); + let data_server = config_with_tls(tmp_dir.path()) + .data_server() + .clone() + .unwrap(); + let server_config = data_server.into_tls().unwrap(); BindServer::new_with_tls( "127.0.0.1:8080".parse().unwrap(), @@ -285,7 +281,7 @@ mod tests { let server = Server::bind_addr(addr, cert_key_pair).await.unwrap(); let port = server.local_addr().unwrap().port(); - let data_server = DataServer::new(server, "/data".to_string(), default_cors_config()); + let data_server = DataServer::new(server, default_cors_config()); tokio::spawn(async move { data_server.serve(path).await.unwrap() }); port diff --git a/htsget-axum/src/server/mod.rs b/htsget-axum/src/server/mod.rs index 3fbe7290..eabacce3 100644 --- a/htsget-axum/src/server/mod.rs +++ b/htsget-axum/src/server/mod.rs @@ -10,6 +10,11 @@ use std::time::Duration; use axum::extract::Request; use axum::Router; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::service_info::ServiceInfo; +use htsget_config::tls::TlsServerConfig; +use htsget_config::types::Scheme; +use htsget_search::HtsGet; use http::HeaderValue; use hyper::body::Incoming; use hyper::service::service_fn; @@ -22,12 +27,6 @@ use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, CorsLayer, Expos use tracing::trace; use tracing::{error, warn}; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::ServiceInfo; -use htsget_config::tls::TlsServerConfig; -use htsget_config::types::Scheme; -use htsget_search::HtsGet; - use crate::error::Error::ServerError; use crate::error::Result; use crate::server::data::DataServer; @@ -79,6 +78,10 @@ pub fn configure_cors(cors: CorsConfig) -> CorsLayer { |cors_layer| cors_layer.allow_headers(AllowHeaders::mirror_request()), cors_layer, ); + cors_layer = cors.allow_origins().apply_mirror( + |cors_layer| cors_layer.allow_headers(AllowHeaders::mirror_request()), + cors_layer, + ); cors_layer = cors.allow_headers().apply_list( |cors_layer, headers| cors_layer.allow_headers(headers.clone()), cors_layer, @@ -88,6 +91,10 @@ pub fn configure_cors(cors: CorsConfig) -> CorsLayer { |cors_layer| cors_layer.allow_methods(AllowMethods::mirror_request()), cors_layer, ); + cors_layer = cors.allow_origins().apply_mirror( + |cors_layer| cors_layer.allow_methods(AllowMethods::mirror_request()), + cors_layer, + ); cors_layer = cors.allow_methods().apply_list( |cors_layer, methods| cors_layer.allow_methods(methods.clone()), cors_layer, @@ -150,10 +157,10 @@ impl BindServer { } /// Eagerly bind the address by returning a `DataServer`. - pub async fn bind_data_server(&mut self, serve_at: String) -> Result { + pub async fn bind_data_server(&mut self) -> Result { let server = self.bind_server().await?; - Ok(DataServer::new(server, serve_at, self.cors.clone())) + Ok(DataServer::new(server, self.cors.clone())) } /// Eagerly bind the address by returning a `TicketServer`. @@ -216,7 +223,7 @@ impl Server { let tls_acceptor = tls_acceptor.clone(); trace!("accepting connection"); - let (cnx, addr) = self.listener.accept().await.unwrap(); + let (cnx, addr) = self.listener.accept().await?; tokio::spawn(async move { let Ok(stream) = tls_acceptor.accept(cnx).await else { diff --git a/htsget-axum/src/server/ticket.rs b/htsget-axum/src/server/ticket.rs index 7f965cf5..4be79d84 100644 --- a/htsget-axum/src/server/ticket.rs +++ b/htsget-axum/src/server/ticket.rs @@ -6,8 +6,10 @@ use crate::handlers::{get, post, reads_service_info, variants_service_info}; use crate::server::{configure_cors, AppState, BindServer, Server}; use axum::routing::get; use axum::Router; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::{Config, ServiceInfo, TicketServerConfig}; +use htsget_config::config::advanced::cors::CorsConfig; +use htsget_config::config::service_info::ServiceInfo; +use htsget_config::config::ticket_server::TicketServerConfig; +use htsget_config::config::Config; use htsget_search::HtsGet; use std::net::SocketAddr; use tokio::task::JoinHandle; @@ -91,7 +93,7 @@ where pub async fn join_handle(config: Config) -> Result>> { let service_info = config.service_info().clone(); let ticket_server = BindServer::from(config.ticket_server().clone()) - .bind_ticket_server(config.owned_resolvers(), service_info) + .bind_ticket_server(config.into_locations(), service_info) .await?; info!(address = ?ticket_server.local_addr()?, "ticket server address bound to"); @@ -168,14 +170,13 @@ mod tests { #[async_trait(?Send)] impl TestServer>> for AxumTestServer { async fn get_expected_path(&self) -> String { - let mut bind_data_server = BindServer::from(self.get_config().data_server().clone()); - let server = bind_data_server - .bind_data_server("/data".to_string()) - .await - .unwrap(); + let data_server = self.get_config().data_server().clone().unwrap(); + + let path = data_server.local_path().to_path_buf(); + let mut bind_data_server = BindServer::from(data_server); + let server = bind_data_server.bind_data_server().await.unwrap(); let addr = server.local_addr(); - let path = self.get_config().data_server().local_path().to_path_buf(); tokio::spawn(async move { server.serve(path).await.unwrap() }); expected_url_path(self.get_config(), addr.unwrap()) @@ -218,7 +219,7 @@ mod tests { async fn get_response(&self, request: Request) -> result::Result { let app = TicketServer::router( - self.config.clone().owned_resolvers(), + self.config.clone().into_locations(), self.config.service_info().clone(), self.config.ticket_server().cors().clone(), ); diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs index 6d15c5fa..17e6b14c 100644 --- a/htsget-config/src/config/advanced/allow_guard.rs +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -1,12 +1,15 @@ //! Allow guard configuration. //! -use crate::types::{Class, Fields, Format, Interval, TaggedTypeAll, Tags}; +use crate::resolver::QueryAllowed; +use crate::types::Format::{Bam, Bcf, Cram, Vcf}; +use crate::types::{Class, Fields, Format, Interval, Query, TaggedTypeAll, Tags}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; /// A query guard represents query parameters that can be allowed to storage for a given query. #[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] pub struct AllowGuard { allow_reference_names: ReferenceNames, allow_fields: Fields, @@ -16,9 +19,22 @@ pub struct AllowGuard { allow_interval: Interval, } +impl Default for AllowGuard { + fn default() -> Self { + Self { + allow_formats: vec![Bam, Cram, Vcf, Bcf], + allow_classes: vec![Class::Body, Class::Header], + allow_interval: Default::default(), + allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), + allow_fields: Fields::Tagged(TaggedTypeAll::All), + allow_tags: Tags::Tagged(TaggedTypeAll::All), + } + } +} + /// Reference names that can be matched. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(untagged)] +#[serde(untagged, deny_unknown_fields)] pub enum ReferenceNames { Tagged(TaggedTypeAll), List(HashSet), @@ -74,3 +90,211 @@ impl AllowGuard { &self.allow_tags } } + +impl QueryAllowed for ReferenceNames { + fn query_allowed(&self, query: &Query) -> bool { + match (self, &query.reference_name()) { + (ReferenceNames::Tagged(TaggedTypeAll::All), _) => true, + (ReferenceNames::List(reference_names), Some(reference_name)) => { + reference_names.contains(*reference_name) + } + (ReferenceNames::List(_), None) => false, + } + } +} + +impl QueryAllowed for Fields { + fn query_allowed(&self, query: &Query) -> bool { + match (self, &query.fields()) { + (Fields::Tagged(TaggedTypeAll::All), _) => true, + (Fields::List(self_fields), Fields::List(query_fields)) => { + self_fields.is_subset(query_fields) + } + (Fields::List(_), Fields::Tagged(TaggedTypeAll::All)) => false, + } + } +} + +impl QueryAllowed for Tags { + fn query_allowed(&self, query: &Query) -> bool { + match (self, &query.tags()) { + (Tags::Tagged(TaggedTypeAll::All), _) => true, + (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), + (Tags::List(_), Tags::Tagged(TaggedTypeAll::All)) => false, + } + } +} + +impl QueryAllowed for AllowGuard { + fn query_allowed(&self, query: &Query) -> bool { + self.allow_formats().contains(&query.format()) + && self.allow_classes().contains(&query.class()) + && self + .allow_interval() + .contains(query.interval().start().unwrap_or(u32::MIN)) + && self + .allow_interval() + .contains(query.interval().end().unwrap_or(u32::MAX)) + && self.allow_reference_names().query_allowed(query) + && self.allow_fields().query_allowed(query) + && self.allow_tags().query_allowed(query) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + use crate::types::Class::Header; + + #[test] + fn allow_reference_names_all() { + test_serialize_and_deserialize( + "allow_reference_names = \"All\"", + AllowGuard { + allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_reference_names_list() { + test_serialize_and_deserialize( + "allow_reference_names = [\"chr1\"]", + AllowGuard { + allow_reference_names: ReferenceNames::List(HashSet::from_iter(vec!["chr1".to_string()])), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_fields_all() { + test_serialize_and_deserialize( + "allow_fields = \"All\"", + AllowGuard { + allow_fields: Fields::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_fields_list() { + test_serialize_and_deserialize( + "allow_fields = [\"field\"]", + AllowGuard { + allow_fields: Fields::List(HashSet::from_iter(vec!["field".to_string()])), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_tags_all() { + test_serialize_and_deserialize( + "allow_tags = \"All\"", + AllowGuard { + allow_tags: Tags::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_tags_list() { + test_serialize_and_deserialize( + "allow_tags = [\"tag\"]", + AllowGuard { + allow_tags: Tags::List(HashSet::from_iter(vec!["tag".to_string()])), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_formats() { + test_serialize_and_deserialize( + "allow_formats = [\"BAM\"]", + AllowGuard { + allow_formats: vec![Bam], + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_classes() { + test_serialize_and_deserialize( + "allow_classes = [\"Header\"]", + AllowGuard { + allow_classes: vec![Header], + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn allow_interval() { + test_serialize_and_deserialize( + r#" + allow_interval.start = 0 + allow_interval.end = 100 + "#, + AllowGuard { + allow_interval: Interval::new(Some(0), Some(100)), + ..Default::default() + }, + |result| result, + ); + } + + #[test] + fn query_allowed() { + let guard = AllowGuard { + allow_formats: vec![Bam], + allow_classes: vec![Header], + allow_interval: Interval::new(Some(0), Some(100)), + ..Default::default() + }; + + let query = Query::new_with_default_request("", Bam); + assert!(!guard.query_allowed(&query)); + assert!(!guard.query_allowed(&query.clone().with_format(Cram))); + + assert!(guard.query_allowed(&query.clone().with_class(Header).with_start(1).with_end(50))); + assert!(guard.query_allowed( + &query + .clone() + .with_class(Header) + .with_start(1) + .with_end(50) + .with_tags(Tags::List(HashSet::from_iter(vec!["tag".to_string()]))) + )); + assert!(guard.query_allowed( + &query + .clone() + .with_class(Header) + .with_start(1) + .with_end(50) + .with_fields(Fields::List(HashSet::from_iter(vec!["field".to_string()]))) + )); + + assert!(!guard.query_allowed( + &query + .clone() + .with_class(Header) + .with_start(1) + .with_end(1000) + )); + } +} diff --git a/htsget-config/src/config/advanced/cors.rs b/htsget-config/src/config/advanced/cors.rs index 901cf384..92e6c640 100644 --- a/htsget-config/src/config/advanced/cors.rs +++ b/htsget-config/src/config/advanced/cors.rs @@ -154,15 +154,15 @@ impl Display for HeaderValue { } /// Cors configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] pub struct CorsConfig { allow_credentials: bool, allow_origins: AllowType, allow_headers: AllowType, allow_methods: AllowType, max_age: usize, - expose_headers: AllowType, + expose_headers: AllowType, } impl CorsConfig { @@ -173,7 +173,7 @@ impl CorsConfig { allow_headers: AllowType, allow_methods: AllowType, max_age: usize, - expose_headers: AllowType, + expose_headers: AllowType, ) -> Self { Self { allow_credentials, @@ -211,7 +211,7 @@ impl CorsConfig { } /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { + pub fn expose_headers(&self) -> &AllowType { &self.expose_headers } } @@ -224,66 +224,63 @@ impl Default for CorsConfig { allow_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), allow_methods: AllowType::Tagged(TaggedAllowTypes::Mirror), max_age: CORS_MAX_AGE, - expose_headers: AllowType::Tagged(TaggedAllowTypes::Mirror), + expose_headers: AllowType::Tagged(TaggedTypeAll::All), } } } #[cfg(test)] mod tests { - use std::fmt::Debug; - + use super::*; + use crate::config::tests::test_serialize_and_deserialize; use http::Method; use toml::de::Error; - use super::*; - - fn test_cors_config(input: &str, expected: &T, get_result: F) - where - F: Fn(&CorsConfig) -> &T, - T: Debug + Eq, - { - let config: CorsConfig = toml::from_str(input).unwrap(); - assert_eq!(expected, get_result(&config)); - - let serialized = toml::to_string(&config).unwrap(); - let deserialized = toml::from_str(&serialized).unwrap(); - assert_eq!(expected, get_result(&deserialized)); - } - #[test] fn unit_variant_any_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "allow_methods = \"All\"", - &AllowType::Tagged(TaggedAllowTypes::All), - |config| config.allow_methods(), + CorsConfig { + allow_methods: AllowType::Tagged(TaggedAllowTypes::All), + ..Default::default() + }, + |result| result, ); } #[test] fn unit_variant_mirror_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "allow_origins = \"Mirror\"", - &AllowType::Tagged(TaggedAllowTypes::Mirror), - |config| config.allow_origins(), + CorsConfig { + allow_origins: AllowType::Tagged(TaggedAllowTypes::Mirror), + ..Default::default() + }, + |result| result, ); } #[test] fn list_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "allow_methods = [\"GET\"]", - &AllowType::List(vec![Method::GET]), - |config| config.allow_methods(), + CorsConfig { + allow_methods: AllowType::List(vec![Method::GET]), + ..Default::default() + }, + |result| result, ); } #[test] fn tagged_any_allow_type() { - test_cors_config( + test_serialize_and_deserialize( "expose_headers = \"All\"", - &AllowType::Tagged(TaggedAllowTypes::All), - |config| config.expose_headers(), + CorsConfig { + expose_headers: AllowType::Tagged(TaggedTypeAll::All), + ..Default::default() + }, + |result| result, ); } diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index 58d57629..516b93b5 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -1,11 +1,20 @@ //! Advanced configuration. //! +use serde::{Deserialize, Serialize}; + pub mod allow_guard; pub mod cors; -pub mod file; pub mod regex_location; -#[cfg(feature = "s3-storage")] -pub mod s3; #[cfg(feature = "url-storage")] pub mod url; + +/// Determines which tracing formatting style to use. +#[derive(Debug, Copy, Clone, Serialize, Deserialize, Default)] +pub enum FormattingStyle { + #[default] + Full, + Compact, + Pretty, + Json, +} diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index 98c9f0d8..0767a2c9 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -2,43 +2,19 @@ //! use crate::config::advanced::allow_guard::AllowGuard; -use crate::config::advanced::file::File; -#[cfg(feature = "s3-storage")] -use crate::config::advanced::s3::S3; -#[cfg(feature = "url-storage")] -use crate::config::advanced::url::UrlStorage; +use crate::config::location::LocationEither; +use crate::storage::Backend; use regex::Regex; use serde::{Deserialize, Serialize}; -/// Specify the storage backend to use as config values. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "backend")] -#[non_exhaustive] -pub enum Backend { - #[serde(alias = "local", alias = "LOCAL")] - Local(File), - #[cfg(feature = "s3-storage")] - #[serde(alias = "s3")] - S3(S3), - #[cfg(feature = "url-storage")] - #[serde(alias = "url", alias = "URL")] - Url(UrlStorage), -} - -impl Default for Backend { - fn default() -> Self { - Self::Local(Default::default()) - } -} - /// A regex storage is a storage that matches ids using Regex. #[derive(Serialize, Debug, Clone, Deserialize)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct RegexLocation { #[serde(with = "serde_regex")] regex: Regex, substitution_string: String, - backend: Backend, + location: Backend, guard: Option, } @@ -47,13 +23,13 @@ impl RegexLocation { pub fn new( regex: Regex, substitution_string: String, - backend: Backend, + location: Backend, guard: Option, ) -> Self { Self { regex, substitution_string, - backend, + location, guard, } } @@ -70,7 +46,12 @@ impl RegexLocation { /// Get the storage backend. pub fn backend(&self) -> &Backend { - &self.backend + &self.location + } + + /// Get the allow guard. + pub fn guard(&self) -> Option<&AllowGuard> { + self.guard.as_ref() } } @@ -84,3 +65,82 @@ impl Default for RegexLocation { ) } } + +impl From for LocationEither { + fn from(location: RegexLocation) -> Self { + Self::Regex(location) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn regex_location_file() { + test_serialize_and_deserialize( + r#" + regex = "123-.*" + substitution_string = "123" + "#, + ("123-.*".to_string(), "123".to_string()), + |result: RegexLocation| { + result.backend().as_file().unwrap(); + ( + result.regex().as_str().to_string(), + result.substitution_string().to_string(), + ) + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn regex_location_s3() { + test_serialize_and_deserialize( + r#" + regex = "123-.*" + substitution_string = "123" + location.backend = "S3" + "#, + ("123-.*".to_string(), "123".to_string()), + |result: RegexLocation| { + result.backend().as_s3().unwrap(); + ( + result.regex().as_str().to_string(), + result.substitution_string().to_string(), + ) + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn regex_location_url() { + test_serialize_and_deserialize( + r#" + regex = "123-.*" + substitution_string = "123" + + [location] + backend = "Url" + url = "https://example.com" + "#, + ( + "123-.*".to_string(), + "123".to_string(), + "https://example.com/".to_string(), + ), + |result: RegexLocation| { + let url = result.backend().as_url().unwrap(); + + ( + result.regex().as_str().to_string(), + result.substitution_string().to_string(), + url.url().to_string(), + ) + }, + ); + } +} diff --git a/htsget-config/src/config/advanced/s3.rs b/htsget-config/src/config/advanced/s3.rs deleted file mode 100644 index abc051ba..00000000 --- a/htsget-config/src/config/advanced/s3.rs +++ /dev/null @@ -1,54 +0,0 @@ -#[cfg(feature = "experimental")] -use crate::storage::c4gh::C4GHKeys; -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct S3 { - bucket: String, - endpoint: Option, - path_style: bool, - #[serde(skip_serializing)] - #[cfg(feature = "experimental")] - keys: Option, -} - -impl S3 { - /// Create a new S3 storage. - pub fn new(bucket: String, endpoint: Option, path_style: bool) -> Self { - Self { - bucket, - endpoint, - path_style, - #[cfg(feature = "experimental")] - keys: None, - } - } - - /// Get the bucket. - pub fn bucket(&self) -> &str { - &self.bucket - } - - /// Get the endpoint - pub fn endpoint(&self) -> Option<&str> { - self.endpoint.as_deref() - } - - /// Get the path style - pub fn path_style(&self) -> bool { - self.path_style - } - - #[cfg(feature = "experimental")] - /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { - self.keys = keys; - self - } - - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} diff --git a/htsget-config/src/config/advanced/url.rs b/htsget-config/src/config/advanced/url.rs index 48544aeb..4fb7ba1e 100644 --- a/htsget-config/src/config/advanced/url.rs +++ b/htsget-config/src/config/advanced/url.rs @@ -1,36 +1,45 @@ +use crate::error::Error; +use crate::error::Error::ParseError; +use crate::error::Result; +use crate::storage; #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; use crate::tls::client::TlsClientConfig; +use cfg_if::cfg_if; use http::Uri; +use reqwest::Client; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct UrlStorage { +#[serde(deny_unknown_fields)] +pub struct Url { #[serde(with = "http_serde::uri")] - uri: Uri, - #[serde(with = "http_serde::uri")] - response_uri: Uri, + url: Uri, + #[serde(with = "http_serde::option::uri", default)] + response_url: Option, + #[serde(default = "default_forward_headers")] forward_headers: bool, + #[serde(default)] header_blacklist: Vec, - #[serde(skip_serializing)] + #[serde(skip_serializing, default)] tls: TlsClientConfig, - #[serde(skip_serializing)] #[cfg(feature = "experimental")] + #[serde(skip_serializing, default)] keys: Option, } -impl UrlStorage { +impl Url { /// Create a new url storage. pub fn new( - uri: Uri, - response_uri: Uri, + url: Uri, + response_url: Option, forward_headers: bool, header_blacklist: Vec, tls: TlsClientConfig, ) -> Self { Self { - uri, - response_uri, + url, + response_url, forward_headers, header_blacklist, tls, @@ -39,14 +48,14 @@ impl UrlStorage { } } - /// Get the uri called when resolving the query. - pub fn uri(&self) -> &Uri { - &self.uri + /// Get the url called when resolving the query. + pub fn url(&self) -> &Uri { + &self.url } - /// Get the response uri which is returned to the client. - pub fn response_uri(&self) -> &Uri { - &self.response_uri + /// Get the response url which is returned to the client. + pub fn response_url(&self) -> Option<&Uri> { + self.response_url.as_ref() } /// Whether headers received in a query request should be @@ -73,3 +82,77 @@ impl UrlStorage { self.keys.as_ref() } } + +impl TryFrom for storage::url::Url { + type Error = Error; + + fn try_from(storage: Url) -> Result { + let mut builder = Client::builder(); + + let (certs, identity) = storage.tls.into_inner(); + + if let Some(certs) = certs { + for cert in certs { + builder = builder.add_root_certificate(cert); + } + } + if let Some(identity) = identity { + builder = builder.identity(identity); + } + + let client = builder + .build() + .map_err(|err| ParseError(format!("building url storage client: {}", err)))?; + + let url_storage = Self::new( + storage.url.clone(), + storage.response_url.unwrap_or(storage.url), + storage.forward_headers, + storage.header_blacklist, + client, + ); + + cfg_if! { + if #[cfg(feature = "experimental")] { + Ok(url_storage.set_keys(storage.keys)) + } else { + Ok(url_storage) + } + } + } +} + +fn default_forward_headers() -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + #[test] + fn url_backend() { + test_serialize_and_deserialize( + r#" + url = "https://example.com" + response_url = "https://example.com" + forward_headers = false + header_blacklist = ["Host"] + "#, + ( + "https://example.com/".to_string(), + "https://example.com/".to_string(), + false, + vec!["Host".to_string()], + ), + |result: Url| { + ( + result.url().to_string(), + result.response_url().unwrap().to_string(), + result.forward_headers(), + result.header_blacklist, + ) + }, + ); + } +} diff --git a/htsget-config/src/config/cors.rs b/htsget-config/src/config/cors.rs deleted file mode 100644 index 75f0546b..00000000 --- a/htsget-config/src/config/cors.rs +++ /dev/null @@ -1,298 +0,0 @@ -use std::fmt::{Display, Formatter}; -use std::str::FromStr; - -use http::header::{HeaderName, HeaderValue as HeaderValueInner, InvalidHeaderValue}; -use http::Method; -use serde::de::Error; -use serde::ser::SerializeSeq; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -use crate::config::default_server_origin; -use crate::types::TaggedTypeAll; - -/// The maximum default amount of time a CORS request can be cached for in seconds. -const CORS_MAX_AGE: usize = 86400; - -/// Tagged allow headers for cors config, either Mirror or Any. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub enum TaggedAllowTypes { - #[serde(alias = "mirror", alias = "MIRROR")] - Mirror, - #[serde(alias = "all", alias = "ALL")] - All, -} - -/// Allowed type for cors config which is used to configure cors behaviour. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(untagged)] -pub enum AllowType { - Tagged(Tagged), - #[serde(bound(serialize = "T: Display", deserialize = "T: FromStr, T::Err: Display"))] - #[serde( - serialize_with = "serialize_allow_types", - deserialize_with = "deserialize_allow_types" - )] - List(Vec), -} - -impl AllowType { - /// Apply a function to the builder when the type is a List. - pub fn apply_list(&self, func: F, builder: U) -> U - where - F: FnOnce(U, &Vec) -> U, - { - if let Self::List(list) = self { - func(builder, list) - } else { - builder - } - } - - /// Apply a function to the builder when the type is a List returning a Result. - pub fn try_apply_list(&self, func: F, builder: U) -> Result - where - F: FnOnce(U, &Vec) -> Result, - { - if let Self::List(list) = self { - func(builder, list) - } else { - Ok(builder) - } - } - - /// Apply a function to the builder when the type is tagged. - pub fn apply_tagged(&self, func: F, builder: U, tagged_type: &Tagged) -> U - where - F: FnOnce(U) -> U, - Tagged: Eq, - { - if let Self::Tagged(tagged) = self { - if tagged == tagged_type { - return func(builder); - } - } - - builder - } -} - -impl AllowType { - /// Apply a function to the builder when the type is Mirror. - pub fn apply_mirror(&self, func: F, builder: U) -> U - where - F: FnOnce(U) -> U, - { - self.apply_tagged(func, builder, &TaggedAllowTypes::Mirror) - } - - /// Apply a function to the builder when the type is Any. - pub fn apply_any(&self, func: F, builder: U) -> U - where - F: FnOnce(U) -> U, - { - self.apply_tagged(func, builder, &TaggedAllowTypes::All) - } -} - -impl AllowType { - /// Apply a function to the builder when the type is Any. - pub fn apply_any(&self, func: F, builder: U) -> U - where - F: FnOnce(U) -> U, - { - self.apply_tagged(func, builder, &TaggedTypeAll::All) - } -} - -fn serialize_allow_types(names: &[T], serializer: S) -> Result -where - T: Display, - S: Serializer, -{ - let mut sequence = serializer.serialize_seq(Some(names.len()))?; - for element in names.iter().map(|name| format!("{name}")) { - sequence.serialize_element(&element)?; - } - sequence.end() -} - -fn deserialize_allow_types<'de, D, T>(deserializer: D) -> Result, D::Error> -where - T: FromStr, - T::Err: Display, - D: Deserializer<'de>, -{ - let names: Vec = Deserialize::deserialize(deserializer)?; - names - .into_iter() - .map(|name| T::from_str(&name).map_err(Error::custom)) - .collect() -} - -/// A wrapper around a http HeaderValue which is used to implement FromStr and Display. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HeaderValue(HeaderValueInner); - -impl HeaderValue { - pub fn into_inner(self) -> HeaderValueInner { - self.0 - } -} - -impl FromStr for HeaderValue { - type Err = InvalidHeaderValue; - - fn from_str(header: &str) -> Result { - Ok(HeaderValue(HeaderValueInner::from_str(header)?)) - } -} - -impl Display for HeaderValue { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&String::from_utf8_lossy(self.0.as_ref())) - } -} - -/// Cors configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct CorsConfig { - allow_credentials: bool, - allow_origins: AllowType, - allow_headers: AllowType, - allow_methods: AllowType, - max_age: usize, - expose_headers: AllowType, -} - -impl CorsConfig { - /// Create new cors config. - pub fn new( - allow_credentials: bool, - allow_origins: AllowType, - allow_headers: AllowType, - allow_methods: AllowType, - max_age: usize, - expose_headers: AllowType, - ) -> Self { - Self { - allow_credentials, - allow_origins, - allow_headers, - allow_methods, - max_age, - expose_headers, - } - } - - /// Get allow credentials. - pub fn allow_credentials(&self) -> bool { - self.allow_credentials - } - - /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { - &self.allow_origins - } - - /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { - &self.allow_headers - } - - /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { - &self.allow_methods - } - - /// Get max age. - pub fn max_age(&self) -> usize { - self.max_age - } - - /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { - &self.expose_headers - } -} - -impl Default for CorsConfig { - fn default() -> Self { - Self { - allow_credentials: false, - allow_origins: AllowType::List(vec![HeaderValue(HeaderValueInner::from_static( - default_server_origin(), - ))]), - allow_headers: AllowType::Tagged(TaggedTypeAll::All), - allow_methods: AllowType::Tagged(TaggedTypeAll::All), - max_age: CORS_MAX_AGE, - expose_headers: AllowType::List(vec![]), - } - } -} - -#[cfg(test)] -mod tests { - use std::fmt::Debug; - - use http::Method; - use toml::de::Error; - - use super::*; - - fn test_cors_config(input: &str, expected: &T, get_result: F) - where - F: Fn(&CorsConfig) -> &T, - T: Debug + Eq, - { - let config: CorsConfig = toml::from_str(input).unwrap(); - assert_eq!(expected, get_result(&config)); - - let serialized = toml::to_string(&config).unwrap(); - let deserialized = toml::from_str(&serialized).unwrap(); - assert_eq!(expected, get_result(&deserialized)); - } - - #[test] - fn unit_variant_any_allow_type() { - test_cors_config( - "allow_methods = \"All\"", - &AllowType::Tagged(TaggedTypeAll::All), - |config| config.allow_methods(), - ); - } - - #[test] - fn unit_variant_mirror_allow_type() { - test_cors_config( - "allow_origins = \"Mirror\"", - &AllowType::Tagged(TaggedAllowTypes::Mirror), - |config| config.allow_origins(), - ); - } - - #[test] - fn list_allow_type() { - test_cors_config( - "allow_methods = [\"GET\"]", - &AllowType::List(vec![Method::GET]), - |config| config.allow_methods(), - ); - } - - #[test] - fn tagged_any_allow_type() { - test_cors_config( - "expose_headers = \"All\"", - &AllowType::Tagged(TaggedTypeAll::All), - |config| config.expose_headers(), - ); - } - - #[test] - fn tagged_any_allow_type_err_on_mirror() { - let allow_type_method = "expose_headers = \"Mirror\""; - let config: Result = toml::from_str(allow_type_method); - assert!(config.is_err()); - } -} diff --git a/htsget-config/src/config/data_server.rs b/htsget-config/src/config/data_server.rs index 5d99bfff..a6219f0e 100644 --- a/htsget-config/src/config/data_server.rs +++ b/htsget-config/src/config/data_server.rs @@ -2,15 +2,41 @@ //! use crate::config::advanced::cors::CorsConfig; -use crate::config::{default_localstorage_addr, default_path}; +use crate::storage::file::{default_localstorage_addr, default_path}; use crate::tls::TlsServerConfig; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; +/// Tagged allow headers for cors config, either Mirror or Any. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum DataServerTagged { + #[serde(alias = "none", alias = "NONE", alias = "null")] + None, +} + +/// Whether the data server is enabled or not. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +#[allow(clippy::large_enum_variant)] +pub enum DataServerEnabled { + None(DataServerTagged), + Some(DataServerConfig), +} + +impl DataServerEnabled { + pub fn unwrap(self) -> DataServerConfig { + if let Self::Some(config) = self { + config + } else { + panic!("called `DataServerEnabled::unwrap()` on a `None` value") + } + } +} + /// Configuration for the htsget server. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct DataServerConfig { addr: SocketAddr, local_path: PathBuf, @@ -54,6 +80,10 @@ impl DataServerConfig { pub fn cors(&self) -> &CorsConfig { &self.cors } + + pub fn into_tls(self) -> Option { + self.tls + } } impl Default for DataServerConfig { @@ -68,3 +98,28 @@ impl Default for DataServerConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn data_server() { + test_serialize_and_deserialize( + r#" + addr = "127.0.0.1:8083" + local_path = "path" + cors.max_age = 1 + "#, + ("127.0.0.1:8083".to_string(), "path".to_string(), 1), + |result: DataServerConfig| { + ( + result.addr().to_string(), + result.local_path().to_string_lossy().to_string(), + result.cors.max_age(), + ) + }, + ); + } +} diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index 39443fc2..8caf5b69 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -2,18 +2,62 @@ //! use crate::config::advanced::regex_location::RegexLocation; -use crate::config::location::Backend::{File, Url, S3}; +use crate::storage; +use crate::storage::file::default_authority; +use crate::storage::Backend; +use crate::types::Scheme; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; +#[cfg(feature = "url-storage")] +use {crate::config::advanced::url::Url, crate::error, http::Uri}; + +/// The locations of data. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default, deny_unknown_fields, from = "LocationsOneOrMany")] +pub struct Locations(Vec); + +impl Locations { + pub fn new(locations: Vec) -> Self { + Self(locations) + } + + pub fn as_slice(&self) -> &[LocationEither] { + self.0.as_slice() + } + + pub fn into_inner(self) -> Vec { + self.0 + } + + pub fn as_mut_slice(&mut self) -> &mut [LocationEither] { + self.0.as_mut_slice() + } +} + +impl Default for Locations { + fn default() -> Self { + Self(vec![Default::default()]) + } +} /// Either simple or regex based location #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(untagged)] +#[serde(untagged, deny_unknown_fields)] pub enum LocationEither { Simple(Location), Regex(RegexLocation), } +impl LocationEither { + /// Get the storage backend. + pub fn backend(&self) -> &Backend { + match self { + LocationEither::Simple(location) => location.backend(), + LocationEither::Regex(regex_location) => regex_location.backend(), + } + } +} + impl Default for LocationEither { fn default() -> Self { Self::Simple(Default::default()) @@ -21,8 +65,8 @@ impl Default for LocationEither { } /// Location config. -#[derive(Serialize, Debug, Clone, Default)] -#[serde(default)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default, from = "LocationWrapper", deny_unknown_fields)] pub struct Location { backend: Backend, prefix: String, @@ -35,8 +79,8 @@ impl Location { } /// Get the storage backend. - pub fn backend(&self) -> Backend { - self.backend + pub fn backend(&self) -> &Backend { + &self.backend } /// Get the prefix. @@ -45,37 +89,304 @@ impl Location { } } -#[derive(Serialize, Deserialize, Debug, Clone, Default, Copy)] -pub enum Backend { - #[default] - File, - S3, - Url, +/// Either a single or many locations +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +enum LocationsOneOrMany { + Many(Vec), + One(LocationEither), +} + +impl From for Locations { + fn from(locations: LocationsOneOrMany) -> Self { + match locations { + LocationsOneOrMany::One(location) => Self(vec![location]), + LocationsOneOrMany::Many(locations) => Self(locations), + } + } +} + +/// Deserialize the location from a string with a protocol. +#[derive(Serialize, Debug, Clone, Default)] +#[serde(default, deny_unknown_fields)] +struct StringLocation { + backend: Backend, + prefix: String, +} + +/// Deserialize the location from a map with regular field and values. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default, deny_unknown_fields)] +struct MapLocation { + backend: Backend, + prefix: String, +} + +/// A wrapper around location deserialization that can deserialize either a string +/// or a map. This is required so that default values behave correctly when deserializing +/// the `Location`. For example, if a location string isn't specified, the `Deserialize` +/// implementation for `StringLocation` can't account for this as it gets passed default values +/// which contain map elements. This wrapper allows deserializing using regular semantics by +/// falling back to the regular `MapLocation` derived deserializer. The reason there needs to be a +/// `StringLocation` and `MapLocation` type is so that `Location` can be deserialized using the +/// `from` attribute without recursion. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(untagged, deny_unknown_fields)] +enum LocationWrapper { + String(StringLocation), + Map(MapLocation), +} + +impl From for Location { + fn from(location: LocationWrapper) -> Self { + match location { + LocationWrapper::String(location) => Location::new(location.backend, location.prefix), + LocationWrapper::Map(location) => Location::new(location.backend, location.prefix), + } + } +} + +impl From for LocationEither { + fn from(location: Location) -> Self { + Self::Simple(location) + } } -impl<'de> Deserialize<'de> for Location { - fn deserialize(deserializer: D) -> Result +impl<'de> Deserialize<'de> for StringLocation { + fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { + let split = |s: &str| { + let (s1, s2) = if let Some(split) = s.split_once("/").map(|(s1, s2)| { + ( + s1.to_string(), + s2.strip_suffix('/').unwrap_or(s2).to_string(), + ) + }) { + split + } else { + (s.to_string(), "".to_string()) + }; + + if s1.is_empty() { + Err(Error::custom("cannot have empty location")) + } else { + Ok((s1, s2)) + } + }; + let s = String::deserialize(deserializer)?.to_lowercase(); - let endpoint = if s.strip_prefix("file://").is_some() { - File - } else if s.strip_prefix("s3://").is_some() { - S3 - } else if s + if let Some(s) = s.strip_prefix("file://") { + let (path, prefix) = split(s)?; + return Ok(StringLocation { + backend: Backend::File(storage::file::File::new( + Scheme::Http, + default_authority(), + path.to_string(), + )), + prefix, + }); + } + + #[cfg(feature = "s3-storage")] + if let Some(s) = s.strip_prefix("s3://") { + let (bucket, prefix) = split(s)?; + return Ok(StringLocation { + backend: Backend::S3(storage::s3::S3::new(bucket.to_string(), None, false)), + prefix, + }); + } + + #[cfg(feature = "url-storage")] + if let Some(s_stripped) = s .strip_prefix("http://") .or_else(|| s.strip_prefix("https://")) - .is_some() { - Url - } else { - return Err(Error::custom( - "expected file://, s3://, http:// or https:// scheme", - )); - }; + let (mut uri, prefix) = split(s_stripped)?; + + if s.starts_with("http://") { + uri = format!("http://{}", uri); + } + if s.starts_with("https://") { + uri = format!("https://{}", uri); + } + + let uri: Uri = uri.parse().map_err(Error::custom)?; + let url = Url::new(uri.clone(), Some(uri), true, vec![], Default::default()) + .try_into() + .map_err(|err: error::Error| Error::custom(err.to_string()))?; + + return Ok(StringLocation { + backend: Backend::Url(url), + prefix, + }); + } + + Err(Error::custom( + "expected file://, s3://, http:// or https:// scheme", + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; + + #[test] + fn location_single() { + test_serialize_and_deserialize( + r#" + locations = "file://path/prefix1" + "#, + ("path".to_string(), "prefix1".to_string()), + |result: Config| assert_file_location(result), + ); + test_serialize_and_deserialize( + r#" + locations = "file://path/prefix1/" + "#, + ("path".to_string(), "prefix1".to_string()), + |result: Config| assert_file_location(result), + ); + } + + #[test] + fn location_no_prefix() { + test_serialize_and_deserialize( + r#" + locations = "file://path" + "#, + ("path".to_string(), "".to_string()), + |result: Config| assert_file_location(result), + ); + test_serialize_and_deserialize( + r#" + locations = "file://path/" + "#, + ("path".to_string(), "".to_string()), + |result: Config| assert_file_location(result), + ); + } + + #[test] + fn location_file() { + test_serialize_and_deserialize( + r#" + locations = [ "file://path/prefix1", "file://path/prefix2" ] + "#, + ( + "path".to_string(), + "prefix1".to_string(), + "path".to_string(), + "prefix2".to_string(), + ), + |result: Config| { + let result = result.locations.0; + assert_eq!(result.len(), 2); + if let (LocationEither::Simple(location1), LocationEither::Simple(location2)) = + (result.first().unwrap(), result.get(1).unwrap()) + { + let file1 = location1.backend().as_file().unwrap(); + let file2 = location2.backend().as_file().unwrap(); + + return ( + file1.local_path().to_string(), + location1.prefix().to_string(), + file2.local_path().to_string(), + location2.prefix().to_string(), + ); + } + + panic!(); + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn location_s3() { + test_serialize_and_deserialize( + r#" + locations = [ "s3://bucket/prefix1", "s3://bucket/prefix2" ] + "#, + ( + "bucket".to_string(), + "prefix1".to_string(), + "bucket".to_string(), + "prefix2".to_string(), + ), + |result: Config| { + let result = result.locations.0; + assert_eq!(result.len(), 2); + if let (LocationEither::Simple(location1), LocationEither::Simple(location2)) = + (result.first().unwrap(), result.get(1).unwrap()) + { + if let (Backend::S3(s31), Backend::S3(s32)) = (location1.backend(), location2.backend()) { + return ( + s31.bucket().to_string(), + location1.prefix().to_string(), + s32.bucket().to_string(), + location2.prefix().to_string(), + ); + } + } + + panic!(); + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn location_url() { + test_serialize_and_deserialize( + r#" + locations = [ "https://example.com/prefix1", "http://example.com/prefix2" ] + "#, + ( + "https://example.com/".to_string(), + "prefix1".to_string(), + "http://example.com/".to_string(), + "prefix2".to_string(), + ), + |result: Config| { + let result = result.locations.0; + assert_eq!(result.len(), 2); + if let (LocationEither::Simple(location1), LocationEither::Simple(location2)) = + (result.first().unwrap(), result.get(1).unwrap()) + { + if let (Backend::Url(url1), Backend::Url(url2)) = + (location1.backend(), location2.backend()) + { + return ( + url1.url().to_string(), + location1.prefix().to_string(), + url2.url().to_string(), + location2.prefix().to_string(), + ); + } + } + + panic!(); + }, + ); + } + + fn assert_file_location(result: Config) -> (String, String) { + let result = result.locations.0; + assert_eq!(result.len(), 1); + if let LocationEither::Simple(location1) = result.first().unwrap() { + let file1 = location1.backend().as_file().unwrap(); + return ( + file1.local_path().to_string(), + location1.prefix().to_string(), + ); + } - Ok(Location::new(endpoint, s)) + panic!(); } } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 95f9f8ee..06b71f12 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -1,56 +1,34 @@ use std::fmt::Debug; use std::io; -use std::net::SocketAddr; use std::path::{Path, PathBuf}; +use crate::config::advanced::FormattingStyle; +use crate::config::data_server::DataServerEnabled; +use crate::config::location::{Location, LocationEither, Locations}; +use crate::config::parser::from_path; +use crate::config::service_info::ServiceInfo; +use crate::config::ticket_server::TicketServerConfig; +use crate::error::Error::{ArgParseError, TracingError}; +use crate::error::Result; +use crate::storage::Backend; use clap::{Args as ClapArgs, Command, FromArgMatches, Parser}; -use http::header::HeaderName; -use http::Method; use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; -use tracing::instrument; use tracing::subscriber::set_global_default; use tracing_subscriber::fmt::{format, layer}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{EnvFilter, Registry}; -use crate::config::cors::{AllowType, CorsConfig, HeaderValue, TaggedAllowTypes}; -use crate::config::parser::from_path; -use crate::config::FormattingStyle::{Compact, Full, Json, Pretty}; -use crate::error::Error::{ArgParseError, TracingError}; -use crate::error::Result; -use crate::resolver::Resolver; -use crate::tls::TlsServerConfig; - pub mod advanced; -pub mod cors; pub mod data_server; pub mod location; pub mod parser; pub mod service_info; -pub mod simple; pub mod ticket_server; /// Represents a usage string for htsget-rs. pub const USAGE: &str = "To configure htsget-rs use a config file or environment variables. \ See the documentation of the htsget-config crate for more information."; -pub(crate) fn default_localstorage_addr() -> &'static str { - "127.0.0.1:8081" -} - -fn default_addr() -> &'static str { - "127.0.0.1:8080" -} - -fn default_server_origin() -> &'static str { - "http://localhost:8080" -} - -pub(crate) fn default_path() -> &'static str { - "./" -} - /// The command line arguments allowed for the htsget-rs executables. #[derive(Parser, Debug)] #[command(author, version, about, long_about = USAGE)] @@ -66,323 +44,62 @@ struct Args { print_default_config: bool, } -/// Determines which tracing formatting style to use. -#[derive(Debug, Copy, Clone, Serialize, Deserialize, Default)] -pub enum FormattingStyle { - #[default] - Full, - Compact, - Pretty, - Json, -} - -with_prefix!(ticket_server_prefix "ticket_server_"); -with_prefix!(data_server_prefix "data_server_"); -with_prefix!(cors_prefix "cors_"); - -/// Configuration for the htsget server. +/// Simplified config. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct Config { - formatting_style: FormattingStyle, - #[serde(flatten, with = "ticket_server_prefix")] ticket_server: TicketServerConfig, - #[serde(flatten, with = "data_server_prefix")] - data_server: DataServerConfig, - #[serde(flatten)] + data_server: DataServerEnabled, service_info: ServiceInfo, - resolvers: Vec, -} - -/// Configuration for the htsget ticket server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct TicketServerConfig { - addr: SocketAddr, - #[serde(skip_serializing)] - tls: Option, - #[serde(flatten, with = "cors_prefix")] - cors: CorsConfig, -} - -impl TicketServerConfig { - /// Create a new ticket server config. - pub fn new(addr: SocketAddr, tls: Option, cors: CorsConfig) -> Self { - Self { addr, tls, cors } - } - - /// Get the addr. - pub fn addr(&self) -> SocketAddr { - self.addr - } - - /// Get the TLS config. - pub fn tls(&self) -> Option<&TlsServerConfig> { - self.tls.as_ref() - } - - /// Get the TLS config. - pub fn into_tls(self) -> Option { - self.tls - } - - /// Get cors config. - pub fn cors(&self) -> &CorsConfig { - &self.cors - } - - /// Get allow credentials. - pub fn allow_credentials(&self) -> bool { - self.cors.allow_credentials() - } - - /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { - self.cors.allow_origins() - } - - /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { - self.cors.allow_headers() - } - - /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { - self.cors.allow_methods() - } - - /// Get max age. - pub fn max_age(&self) -> usize { - self.cors.max_age() - } - - /// Get expose headers. - pub fn expose_headers(&self) -> &AllowType { - self.cors.expose_headers() - } -} - -/// Configuration for the htsget server. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct DataServerConfig { - enabled: bool, - addr: SocketAddr, - local_path: PathBuf, - serve_at: String, - #[serde(skip_serializing)] - tls: Option, - #[serde(flatten, with = "cors_prefix")] - cors: CorsConfig, + locations: Locations, + formatting_style: FormattingStyle, } -impl DataServerConfig { - /// Create a new data server config. +impl Config { + /// Create a config. pub fn new( - enabled: bool, - addr: SocketAddr, - local_path: PathBuf, - serve_at: String, - tls: Option, - cors: CorsConfig, + formatting_style: FormattingStyle, + ticket_server: TicketServerConfig, + data_server: DataServerEnabled, + service_info: ServiceInfo, + locations: Locations, ) -> Self { Self { - enabled, - addr, - local_path, - serve_at, - tls, - cors, - } - } - - /// Get the address. - pub fn addr(&self) -> SocketAddr { - self.addr - } - - /// Get the local path. - pub fn local_path(&self) -> &Path { - &self.local_path - } - - /// Get the serve at path. - pub fn serve_at(&self) -> &str { - &self.serve_at - } - - /// Get the TLS config. - pub fn tls(&self) -> Option<&TlsServerConfig> { - self.tls.as_ref() - } - - /// Get the TLS config. - pub fn into_tls(self) -> Option { - self.tls - } - - /// Get cors config. - pub fn cors(&self) -> &CorsConfig { - &self.cors - } - - /// Get allow credentials. - pub fn allow_credentials(&self) -> bool { - self.cors.allow_credentials() - } - - /// Get allow origins. - pub fn allow_origins(&self) -> &AllowType { - self.cors.allow_origins() - } - - /// Get allow headers. - pub fn allow_headers(&self) -> &AllowType { - self.cors.allow_headers() - } - - /// Get allow methods. - pub fn allow_methods(&self) -> &AllowType { - self.cors.allow_methods() - } - - /// Get the max age. - pub fn max_age(&self) -> usize { - self.cors.max_age() - } - - /// Get the expose headers. - pub fn expose_headers(&self) -> &AllowType { - self.cors.expose_headers() - } - - /// Is the data server disabled - pub fn enabled(&self) -> bool { - self.enabled - } -} - -impl Default for DataServerConfig { - fn default() -> Self { - Self { - enabled: true, - addr: default_localstorage_addr() - .parse() - .expect("expected valid address"), - local_path: default_path().into(), - serve_at: Default::default(), - tls: None, - cors: CorsConfig::default(), + formatting_style, + ticket_server, + data_server, + service_info, + locations, } } -} - -/// Configuration of the service info. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct ServiceInfo { - id: Option, - name: Option, - version: Option, - organization_name: Option, - organization_url: Option, - contact_url: Option, - documentation_url: Option, - created_at: Option, - updated_at: Option, - environment: Option, -} - -impl ServiceInfo { - /// Get the id. - pub fn id(&self) -> Option<&str> { - self.id.as_deref() - } - - /// Get the name. - pub fn name(&self) -> Option<&str> { - self.name.as_deref() - } - /// Get the version. - pub fn version(&self) -> Option<&str> { - self.version.as_deref() - } - - /// Get the organization name. - pub fn organization_name(&self) -> Option<&str> { - self.organization_name.as_deref() - } - - /// Get the organization url. - pub fn organization_url(&self) -> Option<&str> { - self.organization_url.as_deref() - } - - /// Get the contact url. - pub fn contact_url(&self) -> Option<&str> { - self.contact_url.as_deref() + /// Get the ticket server config. + pub fn formatting_style(&self) -> FormattingStyle { + self.formatting_style } - /// Get the documentation url. - pub fn documentation_url(&self) -> Option<&str> { - self.documentation_url.as_deref() + /// Get the ticket server config. + pub fn ticket_server(&self) -> &TicketServerConfig { + &self.ticket_server } - /// Get created at. - pub fn created_at(&self) -> Option<&str> { - self.created_at.as_deref() + /// Get the data server config. + pub fn data_server(&self) -> &DataServerEnabled { + &self.data_server } - /// Get updated at. - pub fn updated_at(&self) -> Option<&str> { - self.updated_at.as_deref() + /// Get the service info config. + pub fn service_info(&self) -> &ServiceInfo { + &self.service_info } - /// Get environment. - pub fn environment(&self) -> Option<&str> { - self.environment.as_deref() + /// Get the location. + pub fn locations(&self) -> &[LocationEither] { + self.locations.as_slice() } -} -impl Default for TicketServerConfig { - fn default() -> Self { - Self { - addr: default_addr().parse().expect("expected valid address"), - tls: None, - cors: CorsConfig::default(), - } - } -} - -impl Default for Config { - fn default() -> Self { - Self { - formatting_style: Full, - ticket_server: TicketServerConfig::default(), - data_server: DataServerConfig::default(), - service_info: ServiceInfo::default(), - resolvers: vec![Resolver::default()], - } - } -} - -impl Config { - /// Create a new config. - pub fn new( - formatting: FormattingStyle, - ticket_server: TicketServerConfig, - data_server: DataServerConfig, - service_info: ServiceInfo, - resolvers: Vec, - ) -> Self { - Self { - formatting_style: formatting, - ticket_server, - data_server, - service_info, - resolvers, - } + pub fn into_locations(self) -> Locations { + self.locations } /// Parse the command line arguments. Returns the config path, or prints the default config. @@ -412,11 +129,9 @@ impl Config { } /// Read a config struct from a TOML file. - #[instrument] pub fn from_path(path: &Path) -> io::Result { let config: Self = from_path(path)?; - - Ok(config.resolvers_from_data_server_config()) + Ok(config.resolvers_from_data_server_config()?) } /// Setup tracing, using a global subscriber. @@ -426,72 +141,61 @@ impl Config { let subscriber = Registry::default().with(env_filter); match self.formatting_style() { - Full => set_global_default(subscriber.with(layer())), - Compact => set_global_default(subscriber.with(layer().event_format(format().compact()))), - Pretty => set_global_default(subscriber.with(layer().event_format(format().pretty()))), - Json => set_global_default(subscriber.with(layer().event_format(format().json()))), + FormattingStyle::Full => set_global_default(subscriber.with(layer())), + FormattingStyle::Compact => { + set_global_default(subscriber.with(layer().event_format(format().compact()))) + } + FormattingStyle::Pretty => { + set_global_default(subscriber.with(layer().event_format(format().pretty()))) + } + FormattingStyle::Json => { + set_global_default(subscriber.with(layer().event_format(format().json()))) + } } .map_err(|err| TracingError(err.to_string()))?; Ok(()) } - /// Get the formatting style. - pub fn formatting_style(&self) -> FormattingStyle { - self.formatting_style - } - - /// Get the ticket server. - pub fn ticket_server(&self) -> &TicketServerConfig { - &self.ticket_server - } - - /// Get the data server. - pub fn data_server(&self) -> &DataServerConfig { - &self.data_server - } - - /// Get the owned data server. - pub fn into_data_server(self) -> DataServerConfig { - self.data_server - } - - /// Get service info. - pub fn service_info(&self) -> &ServiceInfo { - &self.service_info - } - - /// Get the resolvers. - pub fn resolvers(&self) -> &[Resolver] { - &self.resolvers - } - - /// Get owned resolvers. - pub fn owned_resolvers(self) -> Vec { - self.resolvers - } - /// Set the local resolvers from the data server config. - pub fn resolvers_from_data_server_config(self) -> Self { - let Config { - formatting_style: formatting, - ticket_server, - data_server, - service_info, - mut resolvers, - } = self; - - resolvers + pub fn resolvers_from_data_server_config(mut self) -> Result { + self + .locations + .as_mut_slice() .iter_mut() - .for_each(|resolver| resolver.resolvers_from_data_server_config(&data_server)); + .map(|location| { + if let LocationEither::Simple(simple) = location { + // Fall through only if the backend is File. + if simple.backend().as_file().is_err() { + return Ok(()); + } + + if let DataServerEnabled::Some(ref data_server) = self.data_server { + let prefix = simple.prefix().to_string(); + *location = LocationEither::Simple(Location::new( + Backend::File(data_server.try_into()?), + prefix, + )); + } + } + + Ok(()) + }) + .collect::>>()?; + + Ok(self) + } +} - Self::new( - formatting, - ticket_server, - data_server, - service_info, - resolvers, - ) +impl Default for Config { + fn default() -> Self { + Self { + formatting_style: FormattingStyle::Full, + ticket_server: Default::default(), + data_server: DataServerEnabled::Some(Default::default()), + service_info: Default::default(), + locations: Default::default(), + } } } @@ -499,14 +203,14 @@ impl Config { pub(crate) mod tests { use std::fmt::Display; + use super::*; use crate::config::parser::from_str; - use crate::storage::Storage; use crate::tls::tests::with_test_certificates; - use crate::types::Scheme::Http; + use crate::types::Scheme; use figment::Jail; use http::uri::Authority; - - use super::*; + use serde::de::DeserializeOwned; + use serde_json::json; fn test_config(contents: Option<&str>, env_variables: Vec<(K, V)>, test_fn: F) where @@ -531,12 +235,14 @@ pub(crate) mod tests { test_fn( from_path::(path) .map_err(|err| err.to_string())? - .resolvers_from_data_server_config(), + .resolvers_from_data_server_config() + .map_err(|err| err.to_string())?, ); test_fn( from_str::(contents.unwrap_or("")) .map_err(|err| err.to_string())? - .resolvers_from_data_server_config(), + .resolvers_from_data_server_config() + .map_err(|err| err.to_string())?, ); Ok(()) @@ -559,6 +265,20 @@ pub(crate) mod tests { test_config(Some(contents), Vec::<(&str, &str)>::new(), test_fn); } + pub(crate) fn test_serialize_and_deserialize(input: &str, expected: T, get_result: F) + where + T: Debug + PartialEq, + F: Fn(D) -> T, + D: DeserializeOwned + Serialize + Clone, + { + let config: D = toml::from_str(input).unwrap(); + assert_eq!(expected, get_result(config.clone())); + + let serialized = toml::to_string(&config).unwrap(); + let deserialized = toml::from_str(&serialized).unwrap(); + assert_eq!(expected, get_result(deserialized)); + } + #[test] fn config_ticket_server_addr_env() { test_config_from_env( @@ -577,15 +297,15 @@ pub(crate) mod tests { test_config_from_env( vec![("HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS", true)], |config| { - assert!(config.ticket_server().allow_credentials()); + assert!(config.ticket_server().cors().allow_credentials()); }, ); } #[test] fn config_service_info_id_env() { - test_config_from_env(vec![("HTSGET_ID", "id")], |config| { - assert_eq!(config.service_info().id(), Some("id")); + test_config_from_env(vec![("HTSGET_SERVICE_INFO", "{ id=id }")], |config| { + assert_eq!(config.service_info().as_ref().get("id"), Some(&json!("id"))); }); } @@ -595,23 +315,16 @@ pub(crate) mod tests { vec![("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082")], |config| { assert_eq!( - config.data_server().addr(), + config.data_server().clone().unwrap().addr(), "127.0.0.1:8082".parse().unwrap() ); }, ); } - #[test] - fn config_no_data_server_env() { - test_config_from_env(vec![("HTSGET_DATA_SERVER_ENABLED", "true")], |config| { - assert!(config.data_server().enabled()); - }); - } - #[test] fn config_ticket_server_addr_file() { - test_config_from_file(r#"ticket_server_addr = "127.0.0.1:8082""#, |config| { + test_config_from_file(r#"ticket_server.addr = "127.0.0.1:8082""#, |config| { assert_eq!( config.ticket_server().addr(), "127.0.0.1:8082".parse().unwrap() @@ -621,23 +334,23 @@ pub(crate) mod tests { #[test] fn config_ticket_server_cors_allow_origin_file() { - test_config_from_file(r#"ticket_server_cors_allow_credentials = true"#, |config| { - assert!(config.ticket_server().allow_credentials()); + test_config_from_file(r#"ticket_server.cors.allow_credentials = true"#, |config| { + assert!(config.ticket_server().cors().allow_credentials()); }); } #[test] fn config_service_info_id_file() { - test_config_from_file(r#"id = "id""#, |config| { - assert_eq!(config.service_info().id(), Some("id")); + test_config_from_file(r#"service_info.id = "id""#, |config| { + assert_eq!(config.service_info().as_ref().get("id"), Some(&json!("id"))); }); } #[test] fn config_data_server_addr_file() { - test_config_from_file(r#"data_server_addr = "127.0.0.1:8082""#, |config| { + test_config_from_file(r#"data_server.addr = "127.0.0.1:8082""#, |config| { assert_eq!( - config.data_server().addr(), + config.data_server().clone().unwrap().addr(), "127.0.0.1:8082".parse().unwrap() ); }); @@ -652,12 +365,12 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - data_server_tls.key = "{}" + data_server.tls.key = "{}" "#, key_path.to_string_lossy().escape_default() ), |config| { - assert!(config.data_server().tls().is_none()); + assert!(config.data_server().clone().unwrap().tls().is_none()); }, ); }); @@ -672,15 +385,14 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - data_server_tls.key = "{}" - data_server_tls.cert = "{}" + data_server.tls.key = "{}" + data_server.tls.cert = "{}" "#, key_path.to_string_lossy().escape_default(), cert_path.to_string_lossy().escape_default() ), |config| { - println!("{:?}", config.data_server().tls()); - assert!(config.data_server().tls().is_some()); + assert!(config.data_server().clone().unwrap().tls().is_some()); }, ); }); @@ -698,7 +410,7 @@ pub(crate) mod tests { ("HTSGET_DATA_SERVER_TLS_CERT", cert_path.to_string_lossy()), ], |config| { - assert!(config.data_server().tls().is_some()); + assert!(config.data_server().clone().unwrap().tls().is_some()); }, ); }); @@ -713,7 +425,7 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - ticket_server_tls.key = "{}" + ticket_server.tls.key = "{}" "#, key_path.to_string_lossy().escape_default() ), @@ -733,8 +445,8 @@ pub(crate) mod tests { test_config_from_file( &format!( r#" - ticket_server_tls.key = "{}" - ticket_server_tls.cert = "{}" + ticket_server.tls.key = "{}" + ticket_server.tls.cert = "{}" "#, key_path.to_string_lossy().escape_default(), cert_path.to_string_lossy().escape_default() @@ -765,30 +477,24 @@ pub(crate) mod tests { } #[test] - fn config_no_data_server_file() { - test_config_from_file(r#"data_server_enabled = true"#, |config| { - assert!(config.data_server().enabled()); - }); - } - - #[test] - fn resolvers_from_data_server_config() { + fn locations_from_data_server_config() { test_config_from_file( r#" - data_server_addr = "127.0.0.1:8080" - data_server_local_path = "path" - data_server_serve_at = "/path" - - [[resolvers]] - [resolvers.storage] - backend = "Local" - use_data_server_config = true + data_server.addr = "127.0.0.1:8080" + data_server.local_path = "path" + + [[locations]] + backend = "File" "#, |config| { - assert_eq!(config.resolvers.len(), 1); - - assert!(matches!(config.resolvers.first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.local_path() == "path" && local_storage.scheme() == Http && local_storage.authority() == &Authority::from_static("127.0.0.1:8080") && local_storage.path_prefix() == "/path")); + assert_eq!(config.locations().len(), 1); + if let LocationEither::Simple(regex) = config.locations.into_inner().first().unwrap() { + println!("{:#?}", regex); + assert!(matches!(regex.backend(), + Backend::File(file) if file.local_path() == "path" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); + } else { + panic!(); + } }, ); } diff --git a/htsget-config/src/config/parser.rs b/htsget-config/src/config/parser.rs index 3734b0a4..721ece3d 100644 --- a/htsget-config/src/config/parser.rs +++ b/htsget-config/src/config/parser.rs @@ -6,7 +6,7 @@ use std::fmt::Debug; use std::io; use std::io::ErrorKind; use std::path::Path; -use tracing::{info, instrument}; +use tracing::info; const ENVIRONMENT_VARIABLE_PREFIX: &str = "HTSGET_"; @@ -19,7 +19,6 @@ pub enum Parser<'a> { impl Parser<'_> { /// Deserialize a string or path into a config value using Figment. - #[instrument] pub fn deserialize_config_into(&self) -> io::Result where for<'de> T: Deserialize<'de> + Debug, @@ -29,13 +28,17 @@ impl Parser<'_> { Parser::String(string) => Toml::string(string), Parser::Path(path) => Toml::file(path), }) - .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX).map(|k| match k { - k if k.as_str().to_lowercase().contains("tls_") => { - k.as_str().to_lowercase().replace("tls_", "tls.").into() - } - k => k.into(), + .merge(Env::prefixed(ENVIRONMENT_VARIABLE_PREFIX).map(|k| { + // This has to list all possible nested values to resolve issues with ambiguity when + // deserializing. E.g. see https://github.com/SergioBenitez/Figment/issues/12 + k.as_str() + .to_lowercase() + .replace("ticket_server_", "ticket_server.") + .replace("data_server_", "data_server.") + .replace("cors_", "cors.") + .replace("tls_", "tls.") + .into() })) - .merge(Env::raw()) .extract() .map_err(|err| io::Error::new(ErrorKind::Other, format!("failed to parse config: {err}")))?; @@ -46,7 +49,6 @@ impl Parser<'_> { } /// Read a deserializable config struct from a TOML file. -#[instrument] pub fn from_path(path: &Path) -> io::Result where for<'a> T: Deserialize<'a> + Debug, @@ -55,7 +57,6 @@ where } /// Read a deserializable config struct from a str. -#[instrument] pub fn from_str(str: &str) -> io::Result where for<'a> T: Deserialize<'a> + Debug, diff --git a/htsget-config/src/config/service_info.rs b/htsget-config/src/config/service_info.rs index 0e7eafdc..33617cb6 100644 --- a/htsget-config/src/config/service_info.rs +++ b/htsget-config/src/config/service_info.rs @@ -7,21 +7,25 @@ use serde_json::Value; use std::collections::HashMap; /// Service info config. -#[derive(Serialize, Debug, Clone, Default)] -#[serde(default)] -pub struct ServiceInfo { - fields: HashMap, -} +#[derive(Serialize, Debug, Clone, Default, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct ServiceInfo(HashMap); impl ServiceInfo { /// Create a service info. pub fn new(fields: HashMap) -> Self { - Self { fields } + Self(fields) } /// Get the inner value. pub fn into_inner(self) -> HashMap { - self.fields + self.0 + } +} + +impl AsRef> for ServiceInfo { + fn as_ref(&self) -> &HashMap { + &self.0 } } @@ -48,3 +52,24 @@ impl<'de> Deserialize<'de> for ServiceInfo { Ok(ServiceInfo::new(fields)) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + use serde_json::json; + + #[test] + fn service_info() { + test_serialize_and_deserialize( + r#" + id = { "id" = "1" } + "#, + ServiceInfo::new(HashMap::from_iter(vec![( + "id".to_string(), + json!({"id": "1"}), + )])), + |result| result, + ); + } +} diff --git a/htsget-config/src/config/simple.rs b/htsget-config/src/config/simple.rs deleted file mode 100644 index 9856c7ed..00000000 --- a/htsget-config/src/config/simple.rs +++ /dev/null @@ -1,55 +0,0 @@ -//! Simplified configuration. -//! - -use crate::config::data_server::DataServerConfig; -use crate::config::location::LocationEither; -use crate::config::service_info::ServiceInfo; -use crate::config::ticket_server::TicketServerConfig; -use serde::{Deserialize, Serialize}; - -/// Simplified config. -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct Config { - ticket_server: TicketServerConfig, - data_server: Option, - service_info: Option, - location: LocationEither, -} - -impl Config { - /// Create a config. - pub fn new( - ticket_server: TicketServerConfig, - data_server: Option, - service_info: Option, - location: LocationEither, - ) -> Self { - Self { - ticket_server, - data_server, - service_info, - location, - } - } - - /// Get the ticket server config. - pub fn ticket_server(&self) -> &TicketServerConfig { - &self.ticket_server - } - - /// Get the data server config. - pub fn data_server(&self) -> Option<&DataServerConfig> { - self.data_server.as_ref() - } - - /// Get the service info config. - pub fn service_info(&self) -> Option<&ServiceInfo> { - self.service_info.as_ref() - } - - /// Get the location. - pub fn location(&self) -> &LocationEither { - &self.location - } -} diff --git a/htsget-config/src/config/ticket_server.rs b/htsget-config/src/config/ticket_server.rs index 515eccec..3cc659a0 100644 --- a/htsget-config/src/config/ticket_server.rs +++ b/htsget-config/src/config/ticket_server.rs @@ -2,14 +2,13 @@ //! use crate::config::advanced::cors::CorsConfig; -use crate::config::default_addr; use crate::tls::TlsServerConfig; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; /// Configuration for the htsget ticket server. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct TicketServerConfig { addr: SocketAddr, #[serde(skip_serializing)] @@ -37,6 +36,10 @@ impl TicketServerConfig { pub fn cors(&self) -> &CorsConfig { &self.cors } + + pub fn into_tls(self) -> Option { + self.tls + } } impl Default for TicketServerConfig { @@ -48,3 +51,25 @@ impl Default for TicketServerConfig { } } } + +fn default_addr() -> &'static str { + "127.0.0.1:8080" +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn data_server() { + test_serialize_and_deserialize( + r#" + addr = "127.0.0.1:8083" + cors.max_age = 1 + "#, + ("127.0.0.1:8083".to_string(), 1), + |result: TicketServerConfig| (result.addr().to_string(), result.cors.max_age()), + ); + } +} diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index dab3a434..82c39726 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -1,22 +1,12 @@ -use std::collections::HashSet; -use std::result; - +use crate::config::advanced::regex_location::RegexLocation; +use crate::config::location::{LocationEither, Locations}; +use crate::storage; +use crate::storage::{Backend, ResolvedId}; +use crate::types::{Query, Response, Result}; use async_trait::async_trait; -use regex::{Error, Regex}; -use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; +use std::path::Path; use tracing::instrument; -use crate::config::DataServerConfig; -use crate::storage::local::Local; -#[cfg(feature = "s3-storage")] -use crate::storage::s3::S3; -#[cfg(feature = "url-storage")] -use crate::storage::url::UrlStorageClient; -use crate::storage::{ResolvedId, Storage}; -use crate::types::Format::{Bam, Bcf, Cram, Vcf}; -use crate::types::{Class, Fields, Format, Interval, Query, Response, Result, TaggedTypeAll, Tags}; - /// A trait which matches the query id, replacing the match in the substitution text. pub trait IdResolver { /// Resolve the id, returning the substituted string if there is a match. @@ -26,16 +16,16 @@ pub trait IdResolver { /// A trait for determining the response from `Storage`. #[async_trait] pub trait ResolveResponse { - /// Convert from `LocalStorage`. - async fn from_local(local_storage: &Local, query: &Query) -> Result; + /// Convert from `File`. + async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result; - /// Convert from `S3Storage`. + /// Convert from `S3`. #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3, query: &Query) -> Result; + async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result; - /// Convert from `UrlStorage`. + /// Convert from `Url`. #[cfg(feature = "url-storage")] - async fn from_url(url_storage: &UrlStorageClient, query: &Query) -> Result; + async fn from_url(url_storage: &storage::url::Url, query: &Query) -> Result; } /// A trait which uses storage to resolve requests into responses. @@ -54,18 +44,6 @@ pub trait QueryAllowed { fn query_allowed(&self, query: &Query) -> bool; } -/// A regex storage is a storage that matches ids using Regex. -#[derive(Serialize, Debug, Clone, Deserialize)] -#[serde(default)] -pub struct Resolver { - #[serde(with = "serde_regex")] - regex: Regex, - // Todo: should match guard be allowed as variables inside the substitution string? - substitution_string: String, - storage: Storage, - allow_guard: AllowGuard, -} - /// A type which holds a resolved storage and an resolved id. #[derive(Debug)] pub struct ResolvedStorage { @@ -93,293 +71,48 @@ impl ResolvedStorage { } } -impl ResolvedId {} - -with_prefix!(allow_interval_prefix "allow_interval_"); - -/// A query guard represents query parameters that can be allowed to storage for a given query. -#[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(default)] -pub struct AllowGuard { - allow_reference_names: ReferenceNames, - allow_fields: Fields, - allow_tags: Tags, - allow_formats: Vec, - allow_classes: Vec, - #[serde(flatten, with = "allow_interval_prefix")] - allow_interval: Interval, -} - -/// Reference names that can be matched. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum ReferenceNames { - Tagged(TaggedTypeAll), - List(HashSet), -} - -impl AllowGuard { - /// Create a new allow guard. - pub fn new( - allow_reference_names: ReferenceNames, - allow_fields: Fields, - allow_tags: Tags, - allow_formats: Vec, - allow_classes: Vec, - allow_interval: Interval, - ) -> Self { - Self { - allow_reference_names, - allow_fields, - allow_tags, - allow_formats, - allow_classes, - allow_interval, - } - } - - /// Get allow formats. - pub fn allow_formats(&self) -> &[Format] { - &self.allow_formats - } - - /// Get allow classes. - pub fn allow_classes(&self) -> &[Class] { - &self.allow_classes - } - - /// Get allow interval. - pub fn allow_interval(&self) -> Interval { - self.allow_interval - } - - /// Get allow reference names. - pub fn allow_reference_names(&self) -> &ReferenceNames { - &self.allow_reference_names - } - - /// Get allow fields. - pub fn allow_fields(&self) -> &Fields { - &self.allow_fields - } - - /// Get allow tags. - pub fn allow_tags(&self) -> &Tags { - &self.allow_tags - } - - /// Set the allow reference names. - pub fn with_allow_reference_names(mut self, allow_reference_names: ReferenceNames) -> Self { - self.allow_reference_names = allow_reference_names; - self - } - - /// Set the allow fields. - pub fn with_allow_fields(mut self, allow_fields: Fields) -> Self { - self.allow_fields = allow_fields; - self - } - - /// Set the allow tags. - pub fn with_allow_tags(mut self, allow_tags: Tags) -> Self { - self.allow_tags = allow_tags; - self - } - - /// Set the allow formats. - pub fn with_allow_formats(mut self, allow_formats: Vec) -> Self { - self.allow_formats = allow_formats; - self - } - - /// Set the allow classes. - pub fn with_allow_classes(mut self, allow_classes: Vec) -> Self { - self.allow_classes = allow_classes; - self - } - - /// Set the allow interval. - pub fn with_allow_interval(mut self, allow_interval: Interval) -> Self { - self.allow_interval = allow_interval; - self - } -} - -impl Default for AllowGuard { - fn default() -> Self { - Self { - allow_formats: vec![Bam, Cram, Vcf, Bcf], - allow_classes: vec![Class::Body, Class::Header], - allow_interval: Default::default(), - allow_reference_names: ReferenceNames::Tagged(TaggedTypeAll::All), - allow_fields: Fields::Tagged(TaggedTypeAll::All), - allow_tags: Tags::Tagged(TaggedTypeAll::All), - } - } -} - -impl QueryAllowed for ReferenceNames { - fn query_allowed(&self, query: &Query) -> bool { - match (self, &query.reference_name()) { - (ReferenceNames::Tagged(TaggedTypeAll::All), _) => true, - (ReferenceNames::List(reference_names), Some(reference_name)) => { - reference_names.contains(*reference_name) - } - (ReferenceNames::List(_), None) => false, - } - } -} - -impl QueryAllowed for Fields { - fn query_allowed(&self, query: &Query) -> bool { - match (self, &query.fields()) { - (Fields::Tagged(TaggedTypeAll::All), _) => true, - (Fields::List(self_fields), Fields::List(query_fields)) => { - self_fields.is_subset(query_fields) - } - (Fields::List(_), Fields::Tagged(TaggedTypeAll::All)) => false, - } - } -} - -impl QueryAllowed for Tags { - fn query_allowed(&self, query: &Query) -> bool { - match (self, &query.tags()) { - (Tags::Tagged(TaggedTypeAll::All), _) => true, - (Tags::List(self_tags), Tags::List(query_tags)) => self_tags.is_subset(query_tags), - (Tags::List(_), Tags::Tagged(TaggedTypeAll::All)) => false, - } - } -} - -impl QueryAllowed for AllowGuard { - fn query_allowed(&self, query: &Query) -> bool { - self.allow_formats.contains(&query.format()) - && self.allow_classes.contains(&query.class()) - && self - .allow_interval - .contains(query.interval().start().unwrap_or(u32::MIN)) - && self - .allow_interval - .contains(query.interval().end().unwrap_or(u32::MAX)) - && self.allow_reference_names.query_allowed(query) - && self.allow_fields.query_allowed(query) - && self.allow_tags.query_allowed(query) - } -} - -impl Default for Resolver { - fn default() -> Self { - Self::new(Storage::default(), ".*", "$0", AllowGuard::default()) - .expect("expected valid storage") - } -} - -impl Resolver { - /// Create a new regex storage. - pub fn new( - storage: Storage, - regex: &str, - replacement_string: &str, - allow_guard: AllowGuard, - ) -> result::Result { - Ok(Self { - regex: Regex::new(regex)?, - substitution_string: replacement_string.to_string(), - storage, - allow_guard, - }) - } - - /// Set the local resolvers from the data server config. - pub fn resolvers_from_data_server_config(&mut self, config: &DataServerConfig) { - match self.storage() { - Storage::Local(local) => { - if local.use_data_server_config() { - self.storage = Storage::Local(config.into()); - } - } - #[cfg(feature = "s3-storage")] - Storage::S3(_) => {} - #[cfg(feature = "url-storage")] - Storage::Url(_) => {} - } - } - - /// Get the match associated with the capture group at index `i` using the `regex_match`. - pub fn get_match<'a>(&'a self, i: usize, regex_match: &'a str) -> Option<&'a str> { - Some(self.regex().captures(regex_match)?.get(i)?.as_str()) - } - - /// Get the regex. - pub fn regex(&self) -> &Regex { - &self.regex - } - - /// Get the substitution string. - pub fn substitution_string(&self) -> &str { - &self.substitution_string - } - - /// Get the query guard. - pub fn allow_guard(&self) -> &AllowGuard { - &self.allow_guard - } - - /// Get the storage backend. - pub fn storage(&self) -> &Storage { - &self.storage - } - - /// Get allow formats. - pub fn allow_formats(&self) -> &[Format] { - self.allow_guard.allow_formats() - } - - /// Get allow classes. - pub fn allow_classes(&self) -> &[Class] { - self.allow_guard.allow_classes() - } - - /// Get allow interval. - pub fn allow_interval(&self) -> Interval { - self.allow_guard.allow_interval - } - - /// Get allow reference names. - pub fn allow_reference_names(&self) -> &ReferenceNames { - &self.allow_guard.allow_reference_names - } - - /// Get allow fields. - pub fn allow_fields(&self) -> &Fields { - &self.allow_guard.allow_fields - } - - /// Get allow tags. - pub fn allow_tags(&self) -> &Tags { - &self.allow_guard.allow_tags - } -} - -impl IdResolver for Resolver { +impl IdResolver for LocationEither { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - if self.regex.is_match(query.id()) && self.allow_guard.query_allowed(query) { + let replace = |regex_location: &RegexLocation| { Some(ResolvedId::new( - self - .regex - .replace(query.id(), &self.substitution_string) + regex_location + .regex() + .replace(query.id(), regex_location.substitution_string()) .to_string(), )) - } else { - None + }; + + match self { + LocationEither::Simple(location) => { + if query.id().starts_with(location.prefix()) { + return Some(ResolvedId::new( + Path::new(location.prefix()) + .join(query.id()) + .to_str()? + .to_string(), + )); + } + } + LocationEither::Regex(regex_location) => { + if regex_location.regex().is_match(query.id()) { + if let Some(guard) = regex_location.guard() { + if guard.query_allowed(query) { + return replace(regex_location); + } + } + + return replace(regex_location); + } + } } + + None } } #[async_trait] -impl StorageResolver for Resolver { +impl StorageResolver for LocationEither { #[instrument(level = "trace", skip(self), ret)] async fn resolve_request( &self, @@ -390,41 +123,51 @@ impl StorageResolver for Resolver { query.set_id(resolved_id.into_inner()); - match self.storage() { - Storage::Local(local_storage) => Some(T::from_local(local_storage, query).await), + match self.backend() { + Backend::File(file) => Some(T::from_file(file, query).await), #[cfg(feature = "s3-storage")] - Storage::S3(s3_storage) => { - let first_match = self.get_match(1, &_matched_id); - let mut s3_storage = s3_storage.clone(); - if s3_storage.bucket.is_empty() { - s3_storage.bucket = first_match?.to_string(); - } + Backend::S3(s3) => { + let s3 = if let Self::Regex(regex_location) = self { + if s3.bucket().is_empty() { + let first_match = regex_location + .regex() + .captures(&_matched_id)? + .get(1)? + .as_str() + .to_string(); + &s3.clone().with_bucket(first_match) + } else { + s3 + } + } else { + s3 + }; - Some(T::from_s3(&s3_storage, query).await) + Some(T::from_s3(s3, query).await) } #[cfg(feature = "url-storage")] - Storage::Url(url_storage) => Some(T::from_url(url_storage, query).await), + Backend::Url(url_storage) => Some(T::from_url(url_storage, query).await), } } } -impl IdResolver for &[Resolver] { +impl IdResolver for &[LocationEither] { #[instrument(level = "trace", skip(self), ret)] fn resolve_id(&self, query: &Query) -> Option { - self.iter().find_map(|resolver| resolver.resolve_id(query)) + self.iter().find_map(|location| location.resolve_id(query)) } } #[async_trait] -impl StorageResolver for &[Resolver] { +impl StorageResolver for &[LocationEither] { #[instrument(level = "trace", skip(self), ret)] async fn resolve_request( &self, query: &mut Query, ) -> Option> { - for resolver in self.iter() { - if let Some(resolved_storage) = resolver.resolve_request::(query).await { - return Some(resolved_storage); + for location in self.iter() { + if let Some(location) = location.resolve_request::(query).await { + return Some(location); } } @@ -432,179 +175,207 @@ impl StorageResolver for &[Resolver] { } } +impl IdResolver for Locations { + #[instrument(level = "trace", skip(self), ret)] + fn resolve_id(&self, query: &Query) -> Option { + self.as_slice().resolve_id(query) + } +} + +#[async_trait] +impl StorageResolver for Locations { + #[instrument(level = "trace", skip(self), ret)] + async fn resolve_request( + &self, + query: &mut Query, + ) -> Option> { + self.as_slice().resolve_request::(query).await + } +} + #[cfg(test)] mod tests { + use super::*; + use crate::config::location::Location; + use crate::config::tests::{test_config_from_env, test_config_from_file}; + use crate::storage; + use crate::types::Format::Bam; + use crate::types::Scheme::Http; + use crate::types::Url; use http::uri::Authority; - #[cfg(feature = "url-storage")] + use reqwest::ClientBuilder; + #[cfg(feature = "s3-storage")] use { - crate::storage::url, crate::storage::url::ValidatedUrl, http::Uri as InnerUrl, - reqwest::ClientBuilder, std::str::FromStr, + crate::config::advanced::allow_guard::{AllowGuard, ReferenceNames}, + crate::types::{Class, Fields, Interval, Tags}, + std::collections::HashSet, }; - use crate::config::tests::{test_config_from_env, test_config_from_file}; - #[cfg(feature = "s3-storage")] - use crate::storage::s3::S3; - use crate::types::Scheme::Http; - use crate::types::Url; - - use super::*; - struct TestResolveResponse; #[async_trait] impl ResolveResponse for TestResolveResponse { - async fn from_local(local_storage: &Local, _: &Query) -> Result { + async fn from_file(file: &storage::file::File, query: &Query) -> Result { Ok(Response::new( Bam, - vec![Url::new(local_storage.authority().to_string())], + Self::format_url(file.authority().as_ref(), query.id()), )) } #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3, _: &Query) -> Result { - Ok(Response::new(Bam, vec![Url::new(s3_storage.bucket())])) + async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { + Ok(Response::new( + Bam, + Self::format_url(s3_storage.bucket(), query.id()), + )) } #[cfg(feature = "url-storage")] - async fn from_url(url_storage: &UrlStorageClient, _: &Query) -> Result { + async fn from_url(url: &storage::url::Url, query: &Query) -> Result { Ok(Response::new( Bam, - vec![Url::new(url_storage.url().to_string())], + Self::format_url(url.url().to_string().strip_suffix('/').unwrap(), query.id()), )) } } + impl TestResolveResponse { + fn format_url(prefix: &str, id: &str) -> Vec { + vec![Url::new(format!("{}/{}", prefix, id))] + } + } + #[tokio::test] async fn resolver_resolve_local_request() { - let local_storage = Local::new( + let file = storage::file::File::new( Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), - "/data".to_string(), - false, ); - let resolver = Resolver::new( - Storage::Local(local_storage), - "id", - "$0-test", - AllowGuard::default(), - ) - .unwrap(); - expected_resolved_request(resolver, "127.0.0.1:8080").await; + let regex_location = RegexLocation::new( + "id".parse().unwrap(), + "$0-test".to_string(), + Backend::File(file.clone()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.into()], "127.0.0.1:8080/id-test-1").await; + + let location = Location::new(Backend::File(file), "".to_string()); + expected_resolved_request(vec![location.into()], "127.0.0.1:8080/id-1").await; } #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { - let s3_storage = S3::new("id".to_string(), None, false); - let resolver = Resolver::new( - Storage::S3(s3_storage), - "(id)-1", - "$1-test", - AllowGuard::default(), - ) - .unwrap(); + let s3_storage = storage::s3::S3::new("id2".to_string(), None, false); + let regex_location = RegexLocation::new( + "(id)-1".parse().unwrap(), + "$1-test".to_string(), + Backend::S3(s3_storage.clone()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.into()], "id2/id-test").await; - expected_resolved_request(resolver, "id").await; + let location = Location::new(Backend::S3(s3_storage), "".to_string()); + expected_resolved_request(vec![location.into()], "id2/id-1").await; } #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request() { - let resolver = Resolver::new( - Storage::S3(S3::default()), - "(id)-1", - "$1-test", - AllowGuard::default(), - ) - .unwrap(); + let regex_location = RegexLocation::new( + "(id)-1".parse().unwrap(), + "$1-test".to_string(), + Backend::S3(storage::s3::S3::default()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.clone().into()], "id/id-test").await; - expected_resolved_request(resolver, "id").await; + let regex_location = RegexLocation::new( + "^(id)-(?P.*)$".parse().unwrap(), + "$key".to_string(), + Backend::S3(storage::s3::S3::default()), + Default::default(), + ); + expected_resolved_request(vec![regex_location.clone().into()], "id/1").await; + + let location = Location::new( + Backend::S3(storage::s3::S3::new("bucket".to_string(), None, false)), + "".to_string(), + ); + expected_resolved_request(vec![location.into()], "bucket/id-1").await; } #[cfg(feature = "url-storage")] #[tokio::test] async fn resolver_resolve_url_request() { let client = ClientBuilder::new().build().unwrap(); - let url_storage = UrlStorageClient::new( - ValidatedUrl(url::Url { - inner: InnerUrl::from_str("https://example.com/").unwrap(), - }), - ValidatedUrl(url::Url { - inner: InnerUrl::from_str("https://example.com/").unwrap(), - }), + let url_storage = storage::url::Url::new( + "https://example.com/".parse().unwrap(), + "https://example.com/".parse().unwrap(), true, vec![], client, ); - let resolver = Resolver::new( - Storage::Url(url_storage), - "(id)-1", - "$1-test", - AllowGuard::default(), - ) - .unwrap(); - - expected_resolved_request(resolver, "https://example.com/").await; - } - - #[test] - fn resolver_get_matches() { - let resolver = Resolver::new( - Storage::default(), - "^(id)/(?P.*)$", - "$0", - AllowGuard::default(), + let regex_location = RegexLocation::new( + "(id)-1".parse().unwrap(), + "$1-test".to_string(), + Backend::Url(url_storage.clone()), + Default::default(), + ); + expected_resolved_request( + vec![regex_location.clone().into()], + "https://example.com/id-test", ) - .unwrap(); - let first_match = resolver.get_match(1, "id/key").unwrap(); + .await; - assert_eq!(first_match, "id"); + let location = Location::new(Backend::Url(url_storage), "".to_string()); + expected_resolved_request(vec![location.into()], "https://example.com/id-1").await; } #[test] - fn resolver_get_matches_no_captures() { - let resolver = - Resolver::new(Storage::default(), "^id/id$", "$0", AllowGuard::default()).unwrap(); - let first_match = resolver.get_match(1, "/id/key"); - - assert_eq!(first_match, None); - } + fn resolver_array_resolve_id() { + let resolver = Locations::new(vec![ + RegexLocation::new( + "^(id-1)(.*)$".parse().unwrap(), + "$1-test-1".to_string(), + Default::default(), + Default::default(), + ) + .into(), + RegexLocation::new( + "^(id-2)(.*)$".parse().unwrap(), + "$1-test-2".to_string(), + Default::default(), + Default::default(), + ) + .into(), + ]); - #[test] - fn resolver_resolve_id() { - let resolver = - Resolver::new(Storage::default(), "id", "$0-test", AllowGuard::default()).unwrap(); assert_eq!( resolver - .resolve_id(&Query::new_with_default_request("id", Bam)) + .as_slice() + .resolve_id(&Query::new_with_default_request("id-1", Bam)) .unwrap() .into_inner(), - "id-test" + "id-1-test-1" + ); + assert_eq!( + resolver + .as_slice() + .resolve_id(&Query::new_with_default_request("id-2", Bam)) + .unwrap() + .into_inner(), + "id-2-test-2" ); - } - #[test] - fn resolver_array_resolve_id() { - let resolver = vec![ - Resolver::new( - Storage::default(), - "^(id-1)(.*)$", - "$1-test-1", - AllowGuard::default(), - ) - .unwrap(), - Resolver::new( - Storage::default(), - "^(id-2)(.*)$", - "$1-test-2", - AllowGuard::default(), - ) - .unwrap(), - ]; + let resolver = Locations::new(vec![ + Location::new(Default::default(), "id-1".to_string()).into(), + Location::new(Default::default(), "id-2".to_string()).into(), + ]); assert_eq!( resolver @@ -612,7 +383,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-1", Bam)) .unwrap() .into_inner(), - "id-1-test-1" + "id-1/id-1" ); assert_eq!( resolver @@ -620,7 +391,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-2", Bam)) .unwrap() .into_inner(), - "id-2-test-2" + "id-2/id-2" ); } @@ -628,14 +399,15 @@ mod tests { fn config_resolvers_file() { test_config_from_file( r#" - [[resolvers]] + [[locations]] regex = "regex" "#, |config| { - assert_eq!( - config.resolvers().first().unwrap().regex().as_str(), - "regex" - ); + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.regex().as_str(), "regex"); + } else { + panic!(); + } }, ); } @@ -644,28 +416,30 @@ mod tests { fn config_resolvers_guard_file() { test_config_from_file( r#" - [[resolvers]] + [[locations]] regex = "regex" - [resolvers.allow_guard] + [locations.guard] allow_formats = ["BAM"] "#, |config| { - assert_eq!( - config.resolvers().first().unwrap().allow_formats(), - &vec![Bam] - ); + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.guard().unwrap().allow_formats(), &vec![Bam]); + } else { + panic!(); + } }, ); } #[test] fn config_resolvers_env() { - test_config_from_env(vec![("HTSGET_RESOLVERS", "[{regex=regex}]")], |config| { - assert_eq!( - config.resolvers().first().unwrap().regex().as_str(), - "regex" - ); + test_config_from_env(vec![("HTSGET_LOCATIONS", "[{regex=regex}]")], |config| { + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.regex().as_str(), "regex"); + } else { + panic!(); + } }); } @@ -674,12 +448,12 @@ mod tests { fn config_resolvers_all_options_env() { test_config_from_env( vec![( - "HTSGET_RESOLVERS", + "HTSGET_LOCATIONS", "[{ regex=regex, substitution_string=substitution_string, \ - storage={ backend=S3, bucket=bucket }, \ - allow_guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ - allow_formats=[BAM], allow_classes=[body], allow_interval_start=100, \ - allow_interval_end=1000 } }]", + location={ backend=S3, bucket=bucket }, \ + guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ + allow_formats=[BAM], allow_classes=[body], allow_interval={ start=100, \ + end=1000 } } }]", )], |config| { let allow_guard = AllowGuard::new( @@ -690,9 +464,9 @@ mod tests { vec![Class::Body], Interval::new(Some(100), Some(1000)), ); - let resolver = config.resolvers().first().unwrap(); - let expected_storage = S3::new("bucket".to_string(), None, false); - let Storage::S3(storage) = resolver.storage() else { + let resolver = config.locations().first().unwrap(); + let expected_storage = storage::s3::S3::new("bucket".to_string(), None, false); + let Backend::S3(storage) = resolver.backend() else { panic!(); }; @@ -700,16 +474,20 @@ mod tests { assert_eq!(storage.endpoint(), expected_storage.endpoint()); assert_eq!(storage.path_style(), expected_storage.path_style()); - assert_eq!(resolver.regex().to_string(), "regex"); - assert_eq!(resolver.substitution_string(), "substitution_string"); - assert_eq!(resolver.allow_guard(), &allow_guard); + if let LocationEither::Regex(regex) = config.locations().first().unwrap() { + assert_eq!(regex.regex().to_string(), "regex"); + assert_eq!(regex.substitution_string(), "substitution_string"); + assert_eq!(regex.guard().unwrap(), &allow_guard); + } else { + panic!(); + } }, ); } - async fn expected_resolved_request(resolver: Resolver, expected_id: &str) { + async fn expected_resolved_request(resolver: Vec, expected_id: &str) { assert_eq!( - resolver + Locations::new(resolver) .resolve_request::(&mut Query::new_with_default_request("id-1", Bam)) .await .unwrap() diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index c33c8f8a..34146238 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; /// Local C4GH key storage. #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] pub struct C4GHLocal { private_key: PathBuf, recipient_public_key: PathBuf, @@ -42,7 +43,7 @@ impl TryFrom for C4GHKeys { mod tests { use crate::config::tests::test_config_from_file; use crate::config::Config; - use crate::storage::Storage; + use crate::storage::Backend; use std::fs::copy; use std::path::PathBuf; use tempfile::TempDir; @@ -70,14 +71,14 @@ mod tests { test_config_from_file( &format!( r#" - [[resolvers]] + [[locations]] regex = "regex" - [resolvers.storage] + [locations.location] {} - [resolvers.storage.keys] - location = "Local" + [locations.location.keys] + key_location = "File" private_key = "{}" recipient_public_key = "{}" "#, @@ -86,17 +87,16 @@ mod tests { recipient_public_key.to_string_lossy() ), |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); test_fn(config); }, ); } #[tokio::test] async fn config_local_storage_c4gh() { - test_c4gh_storage_config(r#"backend = "Local""#, |config| { + test_c4gh_storage_config(r#"backend = "File""#, |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.keys().is_some() + config.locations().first().unwrap().backend(), + Backend::File(file) if file.keys().is_some() )); }); } @@ -111,8 +111,8 @@ mod tests { "#, |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(s3_storage) if s3_storage.keys().is_some() + config.locations().first().unwrap().backend(), + Backend::S3(s3) if s3.keys().is_some() )); }, ); @@ -130,8 +130,8 @@ mod tests { "#, |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Url(url_storage) if url_storage.keys().is_some() + config.locations().first().unwrap().backend(), + Backend::Url(url) if url.keys().is_some() )); }, ); diff --git a/htsget-config/src/storage/c4gh/mod.rs b/htsget-config/src/storage/c4gh/mod.rs index b6d93800..972ad6ec 100644 --- a/htsget-config/src/storage/c4gh/mod.rs +++ b/htsget-config/src/storage/c4gh/mod.rs @@ -19,7 +19,7 @@ pub mod secrets_manager; /// Config for Crypt4GH keys. #[derive(Deserialize, Debug, Clone)] -#[serde(try_from = "Location")] +#[serde(try_from = "C4GHKeyLocation", deny_unknown_fields)] pub struct C4GHKeys { // Store a cloneable future so that it can be resolved outside serde. keys: Shared>>>, @@ -59,25 +59,25 @@ impl From for Error { } } -impl TryFrom for C4GHKeys { +impl TryFrom for C4GHKeys { type Error = Error; - fn try_from(location: Location) -> Result { + fn try_from(location: C4GHKeyLocation) -> Result { match location { - Location::Local(local) => local.try_into(), + C4GHKeyLocation::File(file) => file.try_into(), #[cfg(feature = "s3-storage")] - Location::SecretsManager(secrets_manager) => secrets_manager.try_into(), + C4GHKeyLocation::SecretsManager(secrets_manager) => secrets_manager.try_into(), } } } /// The location of C4GH keys. #[derive(Deserialize, Debug, Clone)] -#[serde(tag = "location", deny_unknown_fields)] +#[serde(tag = "key_location", deny_unknown_fields)] #[non_exhaustive] -pub enum Location { - #[serde(alias = "local", alias = "LOCAL")] - Local(C4GHLocal), +pub enum C4GHKeyLocation { + #[serde(alias = "file", alias = "FILE")] + File(C4GHLocal), #[cfg(feature = "s3-storage")] #[serde(alias = "secretsmanager", alias = "SECRETSMANAGER")] SecretsManager(C4GHSecretsManager), diff --git a/htsget-config/src/storage/c4gh/secrets_manager.rs b/htsget-config/src/storage/c4gh/secrets_manager.rs index 030792e1..fba485f5 100644 --- a/htsget-config/src/storage/c4gh/secrets_manager.rs +++ b/htsget-config/src/storage/c4gh/secrets_manager.rs @@ -16,6 +16,7 @@ use tempfile::TempDir; /// C4GH secrets manager key storage. #[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub struct C4GHSecretsManager { private_key: String, recipient_public_key: String, diff --git a/htsget-config/src/config/advanced/file.rs b/htsget-config/src/storage/file.rs similarity index 50% rename from htsget-config/src/config/advanced/file.rs rename to htsget-config/src/storage/file.rs index 8982841f..ace26341 100644 --- a/htsget-config/src/config/advanced/file.rs +++ b/htsget-config/src/storage/file.rs @@ -1,23 +1,30 @@ //! Configuration of local file based storage. //! -use crate::config::{default_localstorage_addr, default_path}; +use crate::config::data_server::DataServerConfig; +use crate::error::Error; +use crate::error::Error::ParseError; +use crate::error::Result; #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; +use crate::tls::KeyPairScheme; use crate::types::Scheme; use http::uri::Authority; use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +pub const PATH_PREFIX: &str = "/data"; /// Local file based storage. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] +#[serde(default, deny_unknown_fields)] pub struct File { scheme: Scheme, #[serde(with = "http_serde::authority")] authority: Authority, local_path: String, - #[serde(skip_serializing)] #[cfg(feature = "experimental")] + #[serde(skip_serializing)] keys: Option, } @@ -64,14 +71,59 @@ impl File { impl Default for File { fn default() -> Self { - Self::new(Scheme::Http, default_authority(), default_local_path()) + Self::new(Scheme::Http, default_authority(), default_path().into()) + } +} + +impl TryFrom<&DataServerConfig> for File { + type Error = Error; + + fn try_from(config: &DataServerConfig) -> Result { + Ok(Self::new( + config.tls().get_scheme(), + Authority::from_str(&config.addr().to_string()).map_err(|err| ParseError(err.to_string()))?, + config.local_path().to_string_lossy().to_string(), + )) } } -fn default_authority() -> Authority { +pub(crate) fn default_authority() -> Authority { Authority::from_static(default_localstorage_addr()) } -fn default_local_path() -> String { - default_path().into() +pub(crate) fn default_localstorage_addr() -> &'static str { + "127.0.0.1:8081" +} + +pub(crate) fn default_path() -> &'static str { + "./" +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::tests::test_serialize_and_deserialize; + + #[test] + fn file_backend() { + test_serialize_and_deserialize( + r#" + scheme = "Https" + authority = "127.0.0.1:8083" + local_path = "path" + "#, + ( + "127.0.0.1:8083".to_string(), + Scheme::Https, + "path".to_string(), + ), + |result: File| { + ( + result.authority.to_string(), + result.scheme, + result.local_path, + ) + }, + ); + } } diff --git a/htsget-config/src/storage/local.rs b/htsget-config/src/storage/local.rs deleted file mode 100644 index a514d916..00000000 --- a/htsget-config/src/storage/local.rs +++ /dev/null @@ -1,180 +0,0 @@ -use std::str::FromStr; - -use http::uri::Authority; -use serde::{Deserialize, Serialize}; - -use crate::config::{default_localstorage_addr, default_path, DataServerConfig}; -#[cfg(feature = "experimental")] -use crate::storage::c4gh::C4GHKeys; -use crate::tls::KeyPairScheme; -use crate::types::Scheme; - -pub(crate) fn default_authority() -> Authority { - Authority::from_static(default_localstorage_addr()) -} - -fn default_local_path() -> String { - default_path().into() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct Local { - scheme: Scheme, - #[serde(with = "http_serde::authority")] - authority: Authority, - local_path: String, - path_prefix: String, - use_data_server_config: bool, - #[serde(skip_serializing)] - #[cfg(feature = "experimental")] - keys: Option, -} - -impl Local { - /// Create a new local storage. - pub fn new( - scheme: Scheme, - authority: Authority, - local_path: String, - path_prefix: String, - use_data_server_config: bool, - ) -> Self { - Self { - scheme, - authority, - local_path, - path_prefix, - use_data_server_config, - #[cfg(feature = "experimental")] - keys: None, - } - } - - /// Get the scheme. - pub fn scheme(&self) -> Scheme { - self.scheme - } - - /// Get the authority. - pub fn authority(&self) -> &Authority { - &self.authority - } - - /// Get the local path. - pub fn local_path(&self) -> &str { - &self.local_path - } - - /// Get the path prefix. - pub fn path_prefix(&self) -> &str { - &self.path_prefix - } - - /// Get whether config should be inherited from the data server config. - pub fn use_data_server_config(&self) -> bool { - self.use_data_server_config - } - - #[cfg(feature = "experimental")] - /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { - self.keys = keys; - self - } - - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} - -impl Default for Local { - fn default() -> Self { - Self::new( - Scheme::Http, - default_authority(), - default_local_path(), - Default::default(), - false, - ) - } -} - -impl From<&DataServerConfig> for Local { - fn from(config: &DataServerConfig) -> Self { - Self::new( - config.tls().get_scheme(), - Authority::from_str(&config.addr().to_string()).expect("expected valid authority"), - config.local_path().to_string_lossy().to_string(), - config.serve_at().to_string(), - true, - ) - } -} - -#[cfg(test)] -mod tests { - use std::net::SocketAddr; - use std::path::PathBuf; - - use crate::config::cors::CorsConfig; - use crate::config::tests::test_config_from_file; - use crate::storage::Storage; - use crate::types::Scheme::Http; - - use super::*; - - #[test] - fn config_storage_local_file() { - test_config_from_file( - r#" - [[resolvers]] - regex = "regex" - - [resolvers.storage] - backend = "Local" - local_path = "path" - scheme = "HTTPS" - path_prefix = "path" - "#, - |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.local_path() == "path" && local_storage.scheme() == Scheme::Https && local_storage.path_prefix() == "path" - )); - }, - ); - } - - #[test] - fn local_storage_from_data_server_config() { - let data_server_config = DataServerConfig::new( - true, - SocketAddr::from_str("127.0.0.1:8080").unwrap(), - PathBuf::from("data"), - "/data".to_string(), - None, - CorsConfig::default(), - ); - let result: Local = (&data_server_config).into(); - let expected = Local::new( - Http, - Authority::from_static("127.0.0.1:8080"), - "data".to_string(), - "/data".to_string(), - true, - ); - - assert_eq!(result.scheme(), expected.scheme()); - assert_eq!(result.authority(), expected.authority()); - assert_eq!(result.local_path(), expected.local_path()); - assert_eq!(result.path_prefix(), expected.path_prefix()); - assert_eq!( - result.use_data_server_config(), - expected.use_data_server_config() - ); - } -} diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index d8d277dc..cbf984ba 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,13 +1,16 @@ -use crate::storage::local::Local; +#[cfg(any(feature = "url-storage", feature = "s3-storage"))] +use crate::error::Error; +use crate::error::Result; +use crate::storage::file::File; #[cfg(feature = "s3-storage")] use crate::storage::s3::S3; #[cfg(feature = "url-storage")] -use crate::storage::url::UrlStorageClient; +use crate::storage::url::Url; use serde::{Deserialize, Serialize}; #[cfg(feature = "experimental")] pub mod c4gh; -pub mod local; +pub mod file; #[cfg(feature = "s3-storage")] pub mod s3; #[cfg(feature = "url-storage")] @@ -31,45 +34,76 @@ impl ResolvedId { /// Specify the storage backend to use as config values. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "backend")] +#[serde(tag = "backend", deny_unknown_fields)] #[non_exhaustive] -pub enum Storage { - #[serde(alias = "local", alias = "LOCAL")] - Local(Local), +pub enum Backend { + #[serde(alias = "file", alias = "FILE")] + File(File), #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] S3(S3), #[cfg(feature = "url-storage")] #[serde(alias = "url", alias = "URL")] - Url(#[serde(skip_serializing)] UrlStorageClient), + Url(Url), } -impl Default for Storage { +impl Backend { + /// Get the file variant and error if it is not `File`. + pub fn as_file(&self) -> Result<&File> { + match self { + Backend::File(file) => Ok(file), + #[cfg(feature = "s3-storage")] + Backend::S3(_) => Err(Error::ParseError("not a `File` variant".to_string())), + #[cfg(feature = "url-storage")] + Backend::Url(_) => Err(Error::ParseError("not a `File` variant".to_string())), + } + } + + /// Get the file variant and error if it is not `S3`. + #[cfg(feature = "s3-storage")] + pub fn as_s3(&self) -> Result<&S3> { + if let Backend::S3(s3) = self { + Ok(s3) + } else { + Err(Error::ParseError("not a `S3` variant".to_string())) + } + } + + /// Get the url variant and error if it is not `Url`. + #[cfg(feature = "url-storage")] + pub fn as_url(&self) -> Result<&Url> { + if let Backend::Url(url) = self { + Ok(url) + } else { + Err(Error::ParseError("not a `File` variant".to_string())) + } + } +} + +impl Default for Backend { fn default() -> Self { - Self::Local(Default::default()) + Self::File(Default::default()) } } #[cfg(test)] pub(crate) mod tests { use crate::config::tests::{test_config_from_env, test_config_from_file}; - - use super::*; + use crate::storage::Backend; #[test] fn config_storage_tagged_local_file() { test_config_from_file( r#" - [[resolvers]] - [resolvers.storage] - backend = "Local" + [[locations]] regex = "regex" + [locations.location] + backend = "File" "#, |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local { .. } + config.locations().first().unwrap().backend(), + Backend::File { .. } )); }, ); @@ -78,14 +112,11 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_local_env() { test_config_from_env( - vec![( - "HTSGET_RESOLVERS", - "[{storage={ backend=Local, use_data_server_config=true}}]", - )], + vec![("HTSGET_LOCATIONS", "[{location={ backend=File }}]")], |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Local { .. } + config.locations().first().unwrap().backend(), + Backend::File { .. } )); }, ); @@ -96,16 +127,15 @@ pub(crate) mod tests { fn config_storage_tagged_s3_file() { test_config_from_file( r#" - [[resolvers]] - [resolvers.storage] - backend = "S3" + [[locations]] regex = "regex" + [locations.location] + backend = "S3" "#, |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(..) + config.locations().first().unwrap().backend(), + Backend::S3(..) )); }, ); @@ -115,11 +145,11 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_s3_env() { test_config_from_env( - vec![("HTSGET_RESOLVERS", "[{storage={ backend=S3 }}]")], + vec![("HTSGET_LOCATIONS", "[{location={ backend=S3 }}]")], |config| { assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(..) + config.locations().first().unwrap().backend(), + Backend::S3(..) )); }, ); diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index 5be71ca6..ea22d2b6 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -2,15 +2,15 @@ use crate::storage::c4gh::C4GHKeys; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug, Default, Clone)] -#[serde(default)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default, deny_unknown_fields)] pub struct S3 { - pub(crate) bucket: String, - pub(crate) endpoint: Option, - pub(crate) path_style: bool, - #[serde(skip_serializing)] + bucket: String, + endpoint: Option, + path_style: bool, #[cfg(feature = "experimental")] - pub(crate) keys: Option, + #[serde(skip_serializing)] + keys: Option, } impl S3 { @@ -30,19 +30,37 @@ impl S3 { &self.bucket } + /// Set the bucket. + pub fn with_bucket(mut self, bucket: String) -> Self { + self.bucket = bucket; + self + } + /// Get the endpoint pub fn endpoint(&self) -> Option<&str> { self.endpoint.as_deref() } + /// Set the endpoint. + pub fn with_endpoint(mut self, endpoint: String) -> Self { + self.endpoint = Some(endpoint); + self + } + /// Get the path style pub fn path_style(&self) -> bool { self.path_style } + /// Set the path style. + pub fn with_path_style(mut self, path_style: bool) -> Self { + self.path_style = path_style; + self + } + #[cfg(feature = "experimental")] /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { + pub fn with_keys(mut self, keys: Option) -> Self { self.keys = keys; self } @@ -56,26 +74,24 @@ impl S3 { #[cfg(test)] mod tests { - use crate::config::tests::test_config_from_file; - use crate::storage::Storage; + use super::*; + use crate::config::tests::test_serialize_and_deserialize; #[test] - fn config_storage_s3_file() { - test_config_from_file( + fn s3_backend() { + test_serialize_and_deserialize( r#" - [[resolvers]] - regex = "regex" - - [resolvers.storage] - backend = "S3" - bucket = "bucket" - "#, - |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::S3(s3_storage) if s3_storage.bucket() == "bucket" - )); + bucket = "bucket" + endpoint = "127.0.0.1:8083" + path_style = true + "#, + ("127.0.0.1:8083".to_string(), "bucket".to_string(), true), + |result: S3| { + ( + result.endpoint.unwrap().to_string(), + result.bucket.to_string(), + result.path_style, + ) }, ); } diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index fe1dbbc1..f576dc4a 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -1,93 +1,31 @@ -use cfg_if::cfg_if; -use http::Uri as InnerUrl; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_with::with_prefix; -use std::str::FromStr; - -use crate::error::Error::ParseError; -use crate::error::{Error, Result}; +use crate::config::advanced; #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; -use crate::storage::local::default_authority; -use crate::tls::client::TlsClientConfig; - -fn default_url() -> InnerUrl { - InnerUrl::from_str(&format!("https://{}", default_authority())).expect("expected valid url") -} - -with_prefix!(client_auth_prefix "client_"); +use http::Uri; +use reqwest::Client; +use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(default)] -pub struct UrlStorage { - url: ValidatedUrl, - response_url: ValidatedUrl, +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(try_from = "advanced::url::Url", deny_unknown_fields)] +pub struct Url { + #[serde(with = "http_serde::uri")] + url: Uri, + #[serde(with = "http_serde::uri")] + response_url: Uri, forward_headers: bool, header_blacklist: Vec, #[serde(skip_serializing)] - tls: TlsClientConfig, - #[serde(skip_serializing)] - #[cfg(feature = "experimental")] - keys: Option, -} - -#[derive(Deserialize, Debug, Clone)] -#[serde(try_from = "UrlStorage")] -pub struct UrlStorageClient { - url: ValidatedUrl, - response_url: ValidatedUrl, - forward_headers: bool, - header_blacklist: Vec, client: Client, #[cfg(feature = "experimental")] + #[serde(skip_serializing)] keys: Option, } -impl TryFrom for UrlStorageClient { - type Error = Error; - - fn try_from(storage: UrlStorage) -> Result { - let mut builder = Client::builder(); - - let (certs, identity) = storage.tls.into_inner(); - - if let Some(certs) = certs { - for cert in certs { - builder = builder.add_root_certificate(cert); - } - } - if let Some(identity) = identity { - builder = builder.identity(identity); - } - - let client = builder - .build() - .map_err(|err| ParseError(format!("building url storage client: {}", err)))?; - - let url_storage = Self::new( - storage.url, - storage.response_url, - storage.forward_headers, - storage.header_blacklist, - client, - ); - - cfg_if! { - if #[cfg(feature = "experimental")] { - Ok(url_storage.set_keys(storage.keys)) - } else { - Ok(url_storage) - } - } - } -} - -impl UrlStorageClient { +impl Url { /// Create a new url storage client. pub fn new( - url: ValidatedUrl, - response_url: ValidatedUrl, + url: Uri, + response_url: Uri, forward_headers: bool, header_blacklist: Vec, client: Client, @@ -104,13 +42,13 @@ impl UrlStorageClient { } /// Get the url called when resolving the query. - pub fn url(&self) -> &InnerUrl { - &self.url.0.inner + pub fn url(&self) -> &Uri { + &self.url } /// Get the response url to return to the client - pub fn response_url(&self) -> &InnerUrl { - &self.response_url.0.inner + pub fn response_url(&self) -> &Uri { + &self.response_url } /// Whether to forward headers in the url tickets. @@ -141,168 +79,3 @@ impl UrlStorageClient { self.keys.as_ref() } } - -/// A wrapper around `http::Uri` type which implements serialize and deserialize. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(transparent)] -pub(crate) struct Url { - #[serde(with = "http_serde::uri")] - pub(crate) inner: InnerUrl, -} - -/// A new type struct on top of `http::Uri` which only allows http or https schemes when deserializing. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(try_from = "Url")] -pub struct ValidatedUrl(pub(crate) Url); - -impl ValidatedUrl { - /// Get the inner url. - pub fn into_inner(self) -> InnerUrl { - self.0.inner - } -} - -impl TryFrom for ValidatedUrl { - type Error = Error; - - fn try_from(url: Url) -> Result { - match url.inner.scheme() { - Some(scheme) if scheme == "http" || scheme == "https" => Ok(Self(url)), - _ => Err(ParseError("url scheme must be http or https".to_string())), - } - } -} - -impl UrlStorage { - /// Create a new url storage. - pub fn new( - url: InnerUrl, - response_url: InnerUrl, - forward_headers: bool, - header_blacklist: Vec, - tls: TlsClientConfig, - ) -> Self { - Self { - url: ValidatedUrl(Url { inner: url }), - response_url: ValidatedUrl(Url { - inner: response_url, - }), - forward_headers, - header_blacklist, - tls, - #[cfg(feature = "experimental")] - keys: None, - } - } - - /// Get the url called when resolving the query. - pub fn url(&self) -> &InnerUrl { - &self.url.0.inner - } - - /// Get the response url which is returned to the client. - pub fn response_url(&self) -> &InnerUrl { - &self.url.0.inner - } - - /// Whether headers received in a query request should be - /// included in the returned data block tickets. - pub fn forward_headers(&self) -> bool { - self.forward_headers - } - - /// Get the tls client config. - pub fn tls(&self) -> &TlsClientConfig { - &self.tls - } - - #[cfg(feature = "experimental")] - /// Set the C4GH keys. - pub fn set_keys(mut self, keys: Option) -> Self { - self.keys = keys; - self - } - - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} - -impl Default for UrlStorage { - fn default() -> Self { - Self::new( - default_url(), - default_url(), - true, - vec![], - TlsClientConfig::default(), - ) - } -} - -#[cfg(test)] -mod tests { - use crate::config::tests::test_config_from_file; - use crate::storage::url::{UrlStorage, UrlStorageClient}; - use crate::storage::Storage; - use crate::tls::client::tests::client_config_from_path; - - use crate::tls::tests::with_test_certificates; - - use super::*; - - #[tokio::test] - async fn test_building_client() { - with_test_certificates(|path, _, _| { - let client_config = client_config_from_path(path); - let url_storage = UrlStorageClient::try_from(UrlStorage::new( - "https://example.com".parse::().unwrap(), - "https://example.com".parse::().unwrap(), - true, - vec![], - client_config, - )); - - assert!(url_storage.is_ok()); - }); - } - - #[test] - fn config_storage_url_file() { - with_test_certificates(|path, _, _| { - let key_path = path.join("key.pem"); - let cert_path = path.join("cert.pem"); - - test_config_from_file( - &format!( - r#" - [[resolvers]] - regex = "regex" - - [resolvers.storage] - backend = "Url" - url = "https://example.com/" - response_url = "https://example.com/" - forward_headers = false - tls.key = "{}" - tls.cert = "{}" - tls.root_store = "{}" - "#, - key_path.to_string_lossy().escape_default(), - cert_path.to_string_lossy().escape_default(), - cert_path.to_string_lossy().escape_default() - ), - |config| { - println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( - config.resolvers().first().unwrap().storage(), - Storage::Url(url_storage) if *url_storage.url() == "https://example.com/" - && !url_storage.forward_headers() - )); - }, - ); - }); - } -} diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs index e925a645..c1094cce 100644 --- a/htsget-config/src/tls/mod.rs +++ b/htsget-config/src/tls/mod.rs @@ -29,7 +29,7 @@ pub trait KeyPairScheme { /// A certificate and key pair used for TLS. Serialization is not implemented because there /// is no way to convert back to a `PathBuf`. #[derive(Deserialize, Debug, Clone)] -#[serde(try_from = "CertificateKeyPairPath")] +#[serde(try_from = "CertificateKeyPairPath", deny_unknown_fields)] pub struct TlsServerConfig { server_config: ServerConfig, } @@ -49,6 +49,7 @@ impl TlsServerConfig { /// The location of a certificate and key pair used for TLS. /// This is the path to the PEM formatted X.509 certificate and private key. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] pub struct CertificateKeyPairPath { cert: PathBuf, key: PathBuf, @@ -202,13 +203,12 @@ pub(crate) mod tests { use std::io::Cursor; use std::path::Path; + use super::*; use rcgen::generate_simple_self_signed; use rustls::crypto::aws_lc_rs; use rustls_pemfile::{certs, pkcs8_private_keys}; use tempfile::TempDir; - use super::*; - #[test] fn test_load_key() { with_test_certificates(|path, key, _| { diff --git a/htsget-config/src/types.rs b/htsget-config/src/types.rs index 872f5352..b1cbc856 100644 --- a/htsget-config/src/types.rs +++ b/htsget-config/src/types.rs @@ -17,7 +17,7 @@ pub type Result = result::Result; /// An enumeration with all the possible formats. #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all(serialize = "UPPERCASE"))] +#[serde(rename_all(serialize = "UPPERCASE"), deny_unknown_fields)] pub enum Format { #[serde(alias = "bam", alias = "BAM")] Bam, @@ -102,7 +102,7 @@ impl Display for Format { /// Class component of htsget response. #[derive(Copy, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -#[serde(rename_all(serialize = "lowercase"))] +#[serde(rename_all(serialize = "lowercase"), deny_unknown_fields)] pub enum Class { #[serde(alias = "header", alias = "HEADER")] Header, @@ -113,6 +113,7 @@ pub enum Class { /// An interval represents the start (0-based, inclusive) and end (0-based exclusive) ranges of the /// query. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct Interval { start: Option, end: Option, @@ -220,7 +221,7 @@ pub enum TaggedTypeAll { /// Possible values for the fields parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(untagged)] +#[serde(untagged, deny_unknown_fields)] pub enum Fields { /// Include all fields Tagged(TaggedTypeAll), @@ -230,7 +231,7 @@ pub enum Fields { /// Possible values for the tags parameter. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(untagged)] +#[serde(untagged, deny_unknown_fields)] pub enum Tags { /// Include all tags Tagged(TaggedTypeAll), diff --git a/htsget-http/Cargo.toml b/htsget-http/Cargo.toml index f4fbb260..6603f12f 100644 --- a/htsget-http/Cargo.toml +++ b/htsget-http/Cargo.toml @@ -19,6 +19,7 @@ default = [] [dependencies] thiserror = "1" serde = { version = "1", features = ["derive"] } +serde_json = "1" http = "1" htsget-search = { version = "0.9.1", path = "../htsget-search", default-features = false } htsget-config = { version = "0.12.0", path = "../htsget-config", default-features = false } diff --git a/htsget-http/src/lib.rs b/htsget-http/src/lib.rs index 3ef99f1c..9f0b242e 100644 --- a/htsget-http/src/lib.rs +++ b/htsget-http/src/lib.rs @@ -2,18 +2,14 @@ use std::result; use std::str::FromStr; pub use error::{HtsGetError, Result}; -pub use htsget_config::config::{ - Config, DataServerConfig, ServiceInfo as ConfigServiceInfo, TicketServerConfig, -}; -pub use htsget_config::storage::Storage; +pub use htsget_config::config::Config; use htsget_config::types::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::types::{Format, Query, Request, Response}; pub use http_core::{get, post}; pub use post_request::{PostRequest, Region}; use query_builder::QueryBuilder; pub use service_info::get_service_info_json; -pub use service_info::get_service_info_with; -pub use service_info::{Htsget, Organisation, ServiceInfo, Type}; +pub use service_info::{Htsget, ServiceInfo, Type}; mod error; mod http_core; @@ -84,14 +80,13 @@ mod tests { use std::collections::HashMap; use std::path::PathBuf; - use http::uri::Authority; - - use htsget_config::storage::local::Local as ConfigLocalStorage; + use htsget_config::storage; use htsget_config::types::{Headers, JsonResponse, Request, Scheme, Url}; use htsget_search::from_storage::HtsGetFromStorage; + use htsget_search::FileStorage; use htsget_search::HtsGet; - use htsget_search::LocalStorage; use htsget_search::Storage; + use http::uri::Authority; use super::*; @@ -271,14 +266,12 @@ mod tests { fn get_searcher() -> impl HtsGet + Clone { HtsGetFromStorage::new(Storage::new( - LocalStorage::new( + FileStorage::new( get_base_path(), - ConfigLocalStorage::new( + storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8081"), "data".to_string(), - "/data".to_string(), - false, ), ) .unwrap(), diff --git a/htsget-http/src/service_info.rs b/htsget-http/src/service_info.rs index 5a529dea..b8af5d2a 100644 --- a/htsget-http/src/service_info.rs +++ b/htsget-http/src/service_info.rs @@ -1,42 +1,33 @@ +use htsget_config::config; +use htsget_config::types::Format; +use htsget_search::HtsGet; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; use tracing::debug; use tracing::instrument; -use htsget_config::types::Format; -use htsget_search::HtsGet; - -use crate::ConfigServiceInfo; use crate::Endpoint; const READS_FORMATS: [&str; 2] = ["BAM", "CRAM"]; const VARIANTS_FORMATS: [&str; 2] = ["VCF", "BCF"]; +const HTSGET_GROUP: &str = "org.ga4gh"; +const HTSGET_ARTIFACT: &str = "htsget"; +const HTSGET_VERSION: &str = "1.3.0"; + /// A struct representing the information that should be present in a service-info response. #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct ServiceInfo { - pub id: String, - pub name: String, - pub version: String, - pub organization: Organisation, + #[serde(flatten)] + pub fields: HashMap, #[serde(rename = "type")] pub service_type: Type, pub htsget: Htsget, - pub contact_url: String, - pub documentation_url: String, - pub created_at: String, - pub updated_at: String, - pub environment: String, } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct Organisation { - pub name: String, - pub url: String, -} - -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Type { pub group: String, @@ -44,6 +35,16 @@ pub struct Type { pub version: String, } +impl Default for Type { + fn default() -> Self { + Self { + group: HTSGET_GROUP.to_string(), + artifact: HTSGET_ARTIFACT.to_string(), + version: HTSGET_VERSION.to_string(), + } + } +} + #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct Htsget { @@ -53,42 +54,37 @@ pub struct Htsget { pub tags_parameters_effective: bool, } -pub fn get_service_info_with( - endpoint: Endpoint, - supported_formats: &[Format], - fields_effective: bool, - tags_effective: bool, -) -> ServiceInfo { - let htsget_info = Htsget { - datatype: match endpoint { - Endpoint::Reads => "reads", - Endpoint::Variants => "variants", - } - .to_string(), - formats: supported_formats - .iter() - .map(|format| format.to_string()) - .filter(|format| match endpoint { - Endpoint::Reads => READS_FORMATS.contains(&format.as_str()), - Endpoint::Variants => VARIANTS_FORMATS.contains(&format.as_str()), - }) - .collect(), - fields_parameters_effective: fields_effective, - tags_parameters_effective: tags_effective, - }; +impl ServiceInfo { + pub fn new( + endpoint: Endpoint, + supported_formats: &[Format], + fields_effective: bool, + tags_effective: bool, + fields: HashMap, + ) -> Self { + let htsget_info = Htsget { + datatype: match endpoint { + Endpoint::Reads => "reads", + Endpoint::Variants => "variants", + } + .to_string(), + formats: supported_formats + .iter() + .map(|format| format.to_string()) + .filter(|format| match endpoint { + Endpoint::Reads => READS_FORMATS.contains(&format.as_str()), + Endpoint::Variants => VARIANTS_FORMATS.contains(&format.as_str()), + }) + .collect(), + fields_parameters_effective: fields_effective, + tags_parameters_effective: tags_effective, + }; - ServiceInfo { - id: "".to_string(), - name: "".to_string(), - version: "".to_string(), - organization: Default::default(), - service_type: Default::default(), - htsget: htsget_info, - contact_url: "".to_string(), - documentation_url: "".to_string(), - created_at: "".to_string(), - updated_at: "".to_string(), - environment: "".to_string(), + Self { + fields, + service_type: Default::default(), + htsget: htsget_info, + } } } @@ -96,55 +92,14 @@ pub fn get_service_info_with( pub fn get_service_info_json( endpoint: Endpoint, searcher: impl HtsGet + Send + Sync + 'static, - config: &ConfigServiceInfo, + config: config::service_info::ServiceInfo, ) -> ServiceInfo { debug!(endpoint = ?endpoint,"getting service-info response for endpoint"); - fill_out_service_info_json( - get_service_info_with( - endpoint, - &searcher.get_supported_formats(), - searcher.are_field_parameters_effective(), - searcher.are_tag_parameters_effective(), - ), - config, + ServiceInfo::new( + endpoint, + &searcher.get_supported_formats(), + searcher.are_field_parameters_effective(), + searcher.are_tag_parameters_effective(), + config.into_inner(), ) } - -/// Fills the service-info json with the data from the server config -fn fill_out_service_info_json( - mut service_info_json: ServiceInfo, - config: &ConfigServiceInfo, -) -> ServiceInfo { - if let Some(id) = config.id() { - service_info_json.id = id.to_string(); - } - if let Some(name) = config.name() { - service_info_json.name = name.to_string(); - } - if let Some(version) = config.version() { - service_info_json.version = version.to_string(); - } - if let Some(organization_name) = config.organization_name() { - service_info_json.organization.name = organization_name.to_string(); - } - if let Some(organization_url) = config.organization_url() { - service_info_json.organization.url = organization_url.to_string(); - } - if let Some(contact_url) = config.contact_url() { - service_info_json.contact_url = contact_url.to_string(); - } - if let Some(documentation_url) = config.documentation_url() { - service_info_json.documentation_url = documentation_url.to_string(); - } - if let Some(created_at) = config.created_at() { - service_info_json.created_at = created_at.to_string(); - } - if let Some(updated_at) = config.updated_at() { - service_info_json.updated_at = updated_at.to_string(); - } - if let Some(environment) = config.environment() { - service_info_json.environment = environment.to_string(); - } - - service_info_json -} diff --git a/htsget-lambda/src/main.rs b/htsget-lambda/src/main.rs index 562c7a73..4ad90c38 100644 --- a/htsget-lambda/src/main.rs +++ b/htsget-lambda/src/main.rs @@ -26,7 +26,7 @@ async fn main() -> Result<(), Error> { let service_info = config.service_info().clone(); let cors = config.ticket_server().cors().clone(); - let router = TicketServer::router(config.owned_resolvers(), service_info, cors); + let router = TicketServer::router(config.into_locations(), service_info, cors); run(router).await } else { diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index aab8c499..54bc6c82 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -6,7 +6,7 @@ use http::uri::Authority; use tokio::runtime::Runtime; use htsget_config::resolver::ResolveResponse; -use htsget_config::storage::local::Local as ConfigLocalStorage; +use htsget_config::storage; use htsget_config::types::Class::Header; use htsget_config::types::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::types::{HtsGetError, Query, Scheme}; @@ -16,13 +16,11 @@ const BENCHMARK_DURATION_SECONDS: u64 = 30; const NUMBER_OF_SAMPLES: usize = 50; async fn perform_query(query: Query) -> Result<(), HtsGetError> { - HtsGetFromStorage::from_local( - &ConfigLocalStorage::new( + HtsGetFromStorage::from_file( + &storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8081"), "../data".to_string(), - "/data".to_string(), - false, ), &query, ) diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index d2d54bc6..ca72580b 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -1,19 +1,7 @@ //! Module providing an implementation of the [HtsGet] trait using a [StorageTrait]. //! -use async_trait::async_trait; -use tracing::debug; -use tracing::instrument; - -use htsget_config::resolver::{ResolveResponse, StorageResolver}; -use htsget_config::storage::local::Local as LocalStorageConfig; -#[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3 as S3StorageConfig; -#[cfg(feature = "url-storage")] -use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; - use crate::search::Search; -use crate::Resolver; use crate::{ bam_search::BamSearch, bcf_search::BcfSearch, @@ -22,7 +10,13 @@ use crate::{ {HtsGet, Query, Response, Result}, }; use crate::{Format, HtsGetError}; +use async_trait::async_trait; +use htsget_config::config::location::Locations; +use htsget_config::resolver::{ResolveResponse, StorageResolver}; +use htsget_config::storage; use htsget_storage::Storage; +use tracing::debug; +use tracing::instrument; /// Implementation of the [HtsGet] trait using a [StorageTrait]. #[derive(Debug, Clone)] @@ -31,14 +25,7 @@ pub struct HtsGetFromStorage { } #[async_trait] -impl HtsGet for Vec { - async fn search(self, query: Query) -> Result { - self.as_slice().search(query).await - } -} - -#[async_trait] -impl HtsGet for &[Resolver] { +impl HtsGet for Locations { async fn search(self, mut query: Query) -> Result { self .resolve_request::(&mut query) @@ -63,24 +50,21 @@ impl HtsGet for HtsGetFromStorage { #[async_trait] impl ResolveResponse for HtsGetFromStorage { - async fn from_local( - local_storage_config: &LocalStorageConfig, - query: &Query, - ) -> Result { - let storage = Storage::from_local(local_storage_config).await?; + async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result { + let storage = Storage::from_file(file_storage).await?; let searcher = HtsGetFromStorage::new(storage); searcher.search(query.clone()).await } #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3StorageConfig, query: &Query) -> Result { + async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { let storage = Storage::from_s3(s3_storage).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await } #[cfg(feature = "url-storage")] - async fn from_url(url_storage_config: &UrlStorageConfig, query: &Query) -> Result { + async fn from_url(url_storage_config: &storage::url::Url, query: &Query) -> Result { let storage = Storage::from_url(url_storage_config).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await @@ -111,16 +95,17 @@ pub(crate) mod tests { htsget_storage::s3::S3Storage, htsget_test::aws_mocks::with_s3_test_server, std::fs::create_dir, }; - use http::uri::Authority; - use tempfile::TempDir; - + use htsget_config::config::location::{Location, LocationEither}; use htsget_config::storage; + use htsget_config::storage::Backend; use htsget_config::types::Class::Body; use htsget_config::types::Scheme::Http; - use htsget_storage::local::LocalStorage; + use htsget_storage::local::FileStorage; #[cfg(feature = "experimental")] use htsget_test::c4gh::decrypt_data; use htsget_test::http::concat::ConcatResponse; + use http::uri::Authority; + use tempfile::TempDir; use crate::bam_search::tests::{ expected_url as bam_expected_url, with_local_storage as with_bam_local_storage, BAM_FILE_NAME, @@ -178,7 +163,7 @@ pub(crate) mod tests { |_, local_storage| async move { let filename = "spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Vcf); - let response = HtsGetFromStorage::from_local(&local_storage, &query).await; + let response = HtsGetFromStorage::from_file(&local_storage, &query).await; assert_eq!(response, expected_vcf_response(filename)); @@ -197,17 +182,14 @@ pub(crate) mod tests { async fn search_resolvers() { with_config_local_storage( |_, local_storage| async { - let resolvers = vec![Resolver::new( - storage::Storage::Local(local_storage), - ".*", - "$0", - Default::default(), - ) - .unwrap()]; + let locations = Locations::new(vec![LocationEither::Simple(Location::new( + Backend::File(local_storage), + "".to_string(), + ))]); let filename = "spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Vcf); - let response = resolvers.search(query).await; + let response = locations.search(query).await; assert_eq!(response, expected_vcf_response(filename)); @@ -253,7 +235,7 @@ pub(crate) mod tests { copy_files: &[&str], map: M, ) where - F: FnOnce(PathBuf, LocalStorageConfig) -> Fut, + F: FnOnce(PathBuf, storage::file::File) -> Fut, Fut: Future>, M: FnOnce(&[u8]) -> Vec, { @@ -263,12 +245,10 @@ pub(crate) mod tests { println!("{:#?}", base_path); let response = test( base_path.clone(), - LocalStorageConfig::new( + storage::file::File::new( Http, Authority::from_static("127.0.0.1:8081"), base_path.to_str().unwrap().to_string(), - "/data".to_string(), - false, ), ) .await; @@ -278,7 +258,7 @@ pub(crate) mod tests { async fn with_config_local_storage(test: F, path: &str, copy_files: &[&str]) where - F: FnOnce(PathBuf, LocalStorageConfig) -> Fut, + F: FnOnce(PathBuf, storage::file::File) -> Fut, Fut: Future>, { with_config_local_storage_map(test, path, copy_files, |b| b.to_vec()).await; @@ -308,7 +288,7 @@ pub(crate) mod tests { with_config_local_storage( |base_path, local_storage| async { test(Storage::new( - LocalStorage::new(base_path, local_storage).unwrap(), + FileStorage::new(base_path, local_storage).unwrap(), )) .await }, @@ -327,7 +307,7 @@ pub(crate) mod tests { with_config_local_storage_map( |base_path, local_storage| async { test(Storage::new( - LocalStorage::new(base_path, local_storage).unwrap(), + FileStorage::new(base_path, local_storage).unwrap(), )) .await }, diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 4cd848cf..59d316dd 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -3,17 +3,14 @@ //! Based on the [HtsGet Specification](https://samtools.github.io/hts-specs/htsget.html). //! -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; -pub use htsget_config::resolver::{ - IdResolver, QueryAllowed, ResolveResponse, Resolver, StorageResolver, -}; -pub use htsget_config::storage::Storage as ConfigStorage; +pub use htsget_config::config::Config; +pub use htsget_config::resolver::{IdResolver, QueryAllowed, ResolveResponse, StorageResolver}; pub use htsget_config::types::{ Class, Format, Headers, HtsGetError, JsonResponse, Query, Response, Result, Url, }; pub use htsget_storage::Storage; -pub use htsget_storage::local::LocalStorage; +pub use htsget_storage::local::FileStorage; use std::fmt::Display; use std::str::FromStr; diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index 8e97e9e5..0bc5fac6 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -1,10 +1,7 @@ //! Module providing the abstractions needed to read files from an storage //! -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; -pub use htsget_config::resolver::{ - IdResolver, QueryAllowed, ResolveResponse, Resolver, StorageResolver, -}; +pub use htsget_config::resolver::{IdResolver, QueryAllowed, ResolveResponse, StorageResolver}; pub use htsget_config::types::{ Class, Format, Headers, HtsGetError, JsonResponse, Query, Response, Url, }; @@ -13,7 +10,8 @@ pub use htsget_config::types::{ use crate::c4gh::storage::C4GHStorage; use crate::error::Result; use crate::error::StorageError; -use crate::local::LocalStorage; +use crate::error::StorageError::InvalidKey; +use crate::local::FileStorage; #[cfg(feature = "s3-storage")] use crate::s3::S3Storage; use crate::types::{BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions}; @@ -23,18 +21,16 @@ use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; use cfg_if::cfg_if; +use htsget_config::storage; #[cfg(feature = "experimental")] use htsget_config::storage::c4gh::C4GHKeys; -use htsget_config::storage::local::Local as LocalStorageConfig; -#[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3 as S3StorageConfig; -#[cfg(feature = "url-storage")] -use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; +use htsget_config::storage::file::PATH_PREFIX; use htsget_config::types::Scheme; use http::uri; use pin_project_lite::pin_project; use std::fmt; use std::fmt::{Debug, Formatter}; +use std::path::Path; use std::pin::Pin; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, ReadBuf}; @@ -155,15 +151,12 @@ impl Storage { } /// Create from local storage config. - pub async fn from_local(local_storage: &LocalStorageConfig) -> Result { - let storage = Storage::new(LocalStorage::new( - local_storage.local_path(), - local_storage.clone(), - )?); + pub async fn from_file(file: &storage::file::File) -> Result { + let storage = Storage::new(FileStorage::new(file.local_path(), file.clone())?); cfg_if! { if #[cfg(feature = "experimental")] { - Self::from_c4gh_keys(local_storage.keys(), storage).await + Self::from_c4gh_keys(file.keys(), storage).await } else { Ok(storage) } @@ -172,19 +165,19 @@ impl Storage { /// Create from s3 config. #[cfg(feature = "s3-storage")] - pub async fn from_s3(s3_storage: &S3StorageConfig) -> Result { + pub async fn from_s3(s3: &storage::s3::S3) -> Result { let storage = Storage::new( S3Storage::new_with_default_config( - s3_storage.bucket().to_string(), - s3_storage.endpoint().map(str::to_string), - s3_storage.path_style(), + s3.bucket().to_string(), + s3.endpoint().map(str::to_string), + s3.path_style(), ) .await, ); cfg_if! { if #[cfg(feature = "experimental")] { - Self::from_c4gh_keys(s3_storage.keys(), storage).await + Self::from_c4gh_keys(s3.keys(), storage).await } else { Ok(storage) } @@ -193,18 +186,18 @@ impl Storage { /// Create from url config. #[cfg(feature = "url-storage")] - pub async fn from_url(url_storage: &UrlStorageConfig) -> Result { + pub async fn from_url(url: &storage::url::Url) -> Result { let storage = Storage::new(UrlStorage::new( - url_storage.client_cloned(), - url_storage.url().clone(), - url_storage.response_url().clone(), - url_storage.forward_headers(), - url_storage.header_blacklist().to_vec(), + url.client_cloned(), + url.url().clone(), + url.response_url().clone(), + url.forward_headers(), + url.header_blacklist().to_vec(), )); cfg_if! { if #[cfg(feature = "experimental")] { - Self::from_c4gh_keys(url_storage.keys(), storage).await + Self::from_c4gh_keys(url.keys(), storage).await } else { Ok(storage) } @@ -283,15 +276,20 @@ pub trait UrlFormatter { fn format_url>(&self, key: K) -> Result; } -impl UrlFormatter for htsget_config::storage::local::Local { +impl UrlFormatter for storage::file::File { fn format_url>(&self, key: K) -> Result { + let path = Path::new("/").join(PATH_PREFIX).join(key.as_ref()); uri::Builder::new() .scheme(match self.scheme() { Scheme::Http => uri::Scheme::HTTP, Scheme::Https => uri::Scheme::HTTPS, }) .authority(self.authority().to_string()) - .path_and_query(format!("{}/{}", self.path_prefix(), key.as_ref())) + .path_and_query( + path + .to_str() + .ok_or_else(|| InvalidKey("constructing url".to_string()))?, + ) .build() .map_err(|err| StorageError::InvalidUri(err.to_string())) .map(|value| value.to_string()) @@ -302,17 +300,16 @@ impl UrlFormatter for htsget_config::storage::local::Local { mod tests { use http::uri::Authority; - use crate::local::LocalStorage; - use htsget_config::storage::local::Local as ConfigLocalStorage; + use crate::local::FileStorage; use htsget_test::util::default_dir; use super::*; #[test] fn data_url() { - let result = LocalStorage::::new( + let result = FileStorage::::new( default_dir().join("data"), - ConfigLocalStorage::default(), + storage::file::File::default(), ) .unwrap() .data_url(b"Hello World!".to_vec(), Some(Class::Header)); @@ -323,29 +320,25 @@ mod tests { #[test] fn http_formatter_authority() { - let formatter = ConfigLocalStorage::new( + let formatter = storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), - "/data".to_string(), - false, ); test_formatter_authority(formatter, "http"); } #[test] fn https_formatter_authority() { - let formatter = ConfigLocalStorage::new( + let formatter = storage::file::File::new( Scheme::Https, Authority::from_static("127.0.0.1:8080"), "data".to_string(), - "/data".to_string(), - false, ); test_formatter_authority(formatter, "https"); } - fn test_formatter_authority(formatter: ConfigLocalStorage, scheme: &str) { + fn test_formatter_authority(formatter: storage::file::File, scheme: &str) { assert_eq!( formatter.format_url("path").unwrap(), format!("{}://127.0.0.1:8080{}/path", scheme, "/data") diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index 1aacbeb1..b84d68c2 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -20,12 +20,12 @@ use super::{GetOptions, RangeUrlOptions, Result, StorageError}; /// Implementation for the [StorageTrait] trait using the local file system. [T] is the type of the /// server struct, which is used for formatting urls. #[derive(Debug, Clone)] -pub struct LocalStorage { +pub struct FileStorage { base_path: PathBuf, url_formatter: T, } -impl LocalStorage { +impl FileStorage { pub fn new>(base_path: P, url_formatter: T) -> Result { base_path .as_ref() @@ -79,10 +79,10 @@ impl LocalStorage { } #[async_trait] -impl StorageMiddleware for LocalStorage {} +impl StorageMiddleware for FileStorage {} #[async_trait] -impl StorageTrait for LocalStorage { +impl StorageTrait for FileStorage { /// Get the file at the location of the key. #[instrument(level = "debug", skip(self))] async fn get(&self, key: &str, options: GetOptions<'_>) -> Result { @@ -143,14 +143,13 @@ pub(crate) mod tests { use std::future::Future; use std::matches; + use htsget_config::storage; + use htsget_config::types::Scheme; use http::uri::Authority; use tempfile::TempDir; use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::storage::local::Local as ConfigLocalStorage; - use htsget_config::types::Scheme; - use super::*; use crate::types::BytesPosition; use crate::{GetOptions, RangeUrlOptions, StorageError}; @@ -345,15 +344,13 @@ pub(crate) mod tests { (folder_name.to_string(), base_path) } - pub(crate) fn test_local_storage(base_path: &Path) -> LocalStorage { - LocalStorage::new( + pub(crate) fn test_local_storage(base_path: &Path) -> FileStorage { + FileStorage::new( base_path, - ConfigLocalStorage::new( + storage::file::File::new( Scheme::Http, Authority::from_static("127.0.0.1:8081"), "data".to_string(), - "/data".to_string(), - false, ), ) .unwrap() @@ -361,7 +358,7 @@ pub(crate) mod tests { pub(crate) async fn with_local_storage(test: F) where - F: FnOnce(LocalStorage, PathBuf) -> Fut, + F: FnOnce(FileStorage, PathBuf) -> Fut, Fut: Future, { let (_, base_path) = create_local_test_files().await; diff --git a/htsget-test/src/http/mod.rs b/htsget-test/src/http/mod.rs index 43e9b4ef..3aacdffc 100644 --- a/htsget-test/src/http/mod.rs +++ b/htsget-test/src/http/mod.rs @@ -11,21 +11,23 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use async_trait::async_trait; -use http::uri::Authority; -use http::{HeaderMap, HeaderName, Method}; -use serde::de; - -use htsget_config::config::cors::{AllowType, CorsConfig}; -use htsget_config::config::{DataServerConfig, TicketServerConfig}; -use htsget_config::resolver::Resolver; -use htsget_config::storage::{local::Local, Storage}; +use htsget_config::config::advanced::cors::{AllowType, CorsConfig, TaggedAllowTypes}; +use htsget_config::config::advanced::regex_location::RegexLocation; +use htsget_config::config::data_server::{DataServerConfig, DataServerEnabled}; +use htsget_config::config::location::{LocationEither, Locations}; +use htsget_config::config::ticket_server::TicketServerConfig; +use htsget_config::config::Config; +use htsget_config::storage::file::File; +use htsget_config::storage::Backend; use htsget_config::tls::{ load_certs, load_key, tls_server_config, CertificateKeyPair, TlsServerConfig, }; -use htsget_config::types::{Scheme, TaggedTypeAll}; +use htsget_config::types::Scheme; +use http::uri::Authority; +use http::{HeaderMap, HeaderName, Method}; +use serde::de; use crate::util::{default_dir, default_dir_data, generate_test_certificates}; -use crate::Config; /// Represents a http header. #[derive(Debug)] @@ -94,30 +96,27 @@ pub trait TestServer { } /// Get the default test storage. -pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> Vec { - let local_storage = Local::new( +pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> Locations { + let local_storage = File::new( scheme, Authority::from_str(&addr.to_string()).unwrap(), default_dir_data().to_str().unwrap().to_string(), - "/data".to_string(), - false, ); - vec![ - Resolver::new( - Storage::Local(local_storage.clone()), - "^1-(.*)$", - "$1", + + Locations::new(vec![ + LocationEither::Regex(RegexLocation::new( + "^1-(.*)$".parse().unwrap(), + "$1".to_string(), + Backend::File(local_storage.clone()), Default::default(), - ) - .unwrap(), - Resolver::new( - Storage::Local(local_storage), - "^2-(.*)$", - "$1", + )), + LocationEither::Regex(RegexLocation::new( + "^2-(.*)$".parse().unwrap(), + "$1".to_string(), + Backend::File(local_storage.clone()), Default::default(), - ) - .unwrap(), - ] + )), + ]) } /// Default config with fixed port. @@ -137,8 +136,8 @@ pub fn default_cors_config() -> CorsConfig { CorsConfig::new( false, AllowType::List(vec!["http://example.com".parse().unwrap()]), - AllowType::Tagged(TaggedTypeAll::All), - AllowType::Tagged(TaggedTypeAll::All), + AllowType::Tagged(TaggedAllowTypes::All), + AllowType::Tagged(TaggedAllowTypes::All), 1000, AllowType::List(vec![]), ) @@ -150,19 +149,12 @@ fn default_test_config_params( scheme: Scheme, ) -> Config { let cors = default_cors_config(); - let server_config = DataServerConfig::new( - true, - addr, - default_dir_data(), - "/data".to_string(), - tls.clone(), - cors.clone(), - ); + let server_config = DataServerConfig::new(addr, default_dir_data(), tls.clone(), cors.clone()); Config::new( Default::default(), TicketServerConfig::new("127.0.0.1:8080".parse().unwrap(), tls, cors), - server_config, + DataServerEnabled::Some(server_config), Default::default(), default_test_resolver(addr, scheme), ) diff --git a/htsget-test/src/http/server.rs b/htsget-test/src/http/server.rs index 87b98931..818819cd 100644 --- a/htsget-test/src/http/server.rs +++ b/htsget-test/src/http/server.rs @@ -1,17 +1,17 @@ use std::fmt::Debug; use std::net::SocketAddr; +use crate::http::concat::ConcatResponse; +use htsget_config::config::data_server::DataServerEnabled; +use htsget_config::config::Config; +use htsget_config::types::Class; +use htsget_config::types::Format; use http::{HeaderValue, Method, StatusCode}; use reqwest::ClientBuilder; use serde::Deserialize; use serde_json::{json, Value}; -use crate::http::concat::ConcatResponse; -use htsget_config::types::Class; -use htsget_config::types::Format; - use crate::http::{Header, Response, TestRequest, TestServer}; -use crate::Config; /// Test response with with class. pub async fn test_response(response: Response, class: Class) @@ -51,27 +51,22 @@ where /// Get the expected url path from the formatter. pub fn expected_url_path(config: &Config, local_addr: SocketAddr) -> String { - let scheme = match config.data_server().tls() { - None => "http", - Some(_) => "https", - }; + let mut scheme = "http"; + if let DataServerEnabled::Some(server) = config.data_server() { + if server.tls().is_some() { + scheme = "https"; + } + } format!("{}://{}", scheme, local_addr) } /// Test response with with service info. pub fn test_response_service_info(response: &Response) { let expected = json!({ - "id": "", - "name": "", - "version": "", - "organization": { - "name": "", - "url": "", - }, "type": { - "group": "", - "artifact": "", - "version": "", + "group": "org.ga4gh", + "artifact": "htsget", + "version": "1.3.0", }, "htsget": { "datatype": "variants", @@ -82,11 +77,6 @@ pub fn test_response_service_info(response: &Response) { "fieldsParametersEffective": false, "tagsParametersEffective": false, }, - "contactUrl": "", - "documentationUrl": "", - "createdAt": "", - "updatedAt": "", - "environment": "", }); println!("{:#?}", expected); diff --git a/htsget-test/src/lib.rs b/htsget-test/src/lib.rs index d5c10391..817ad8ef 100644 --- a/htsget-test/src/lib.rs +++ b/htsget-test/src/lib.rs @@ -1,9 +1,3 @@ -#[cfg(feature = "http")] -pub use htsget_config::{ - config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}, - storage::Storage, -}; - #[cfg(feature = "aws-mocks")] pub mod aws_mocks; #[cfg(feature = "experimental")] diff --git a/htsget-test/src/util.rs b/htsget-test/src/util.rs index aaeeae99..e4a2f066 100644 --- a/htsget-test/src/util.rs +++ b/htsget-test/src/util.rs @@ -34,7 +34,7 @@ pub fn default_dir() -> PathBuf { .to_path_buf() } -/// Get the default directory where data is present.. +/// Get the default directory where data is present. pub fn default_dir_data() -> PathBuf { default_dir().join("data") } From ac0dc01a0333827975a289f3be30b6e17f60c541 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 16 Dec 2024 10:54:19 +1100 Subject: [PATCH 07/12] fix(config): update file from data server config --- htsget-config/README.md | 26 +++++++- .../examples/config-files/basic.toml | 14 +---- htsget-config/src/config/location.rs | 22 ++++++- htsget-config/src/config/mod.rs | 62 +++++++++++++------ htsget-config/src/storage/file.rs | 6 ++ 5 files changed, 97 insertions(+), 33 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index 4f9ec7d2..6e7d00fd 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -14,6 +14,28 @@ Configuration for [htsget-rs]. ## Overview +The simplest way to use htsget-rs is to create a [toml] config file and specify a storage location: + +```toml +locations = "file://data" +``` + +Then launch the server using the config file: + +```sh +cargo run -p htsget-axum -- --config +``` + +This will serve files under the [`data`][data] directory: + +```sh +curl 'http://localhost:8080/variants/data/vcf/sample1-bcbio-cancer' +``` + +Locations allow htsget-rs access to bioinformatics files and indexes. In this example, htsget-rs can read For example, to give htsget-rs access to the data directory, configure +a file location: + + This crate is used to configure htsget-rs using a config file or environment variables. ## Usage @@ -572,4 +594,6 @@ This project is licensed under the [MIT license][license]. [secrets-manager]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html [id]: https://samtools.github.io/hts-specs/htsget.html#url-parameters [basic]: examples/config-files/basic.toml -[data-server]: README.md#data-server-config \ No newline at end of file +[data-server]: README.md#data-server-config +[toml]: https://toml.io/en/ +[data]: ../data \ No newline at end of file diff --git a/htsget-config/examples/config-files/basic.toml b/htsget-config/examples/config-files/basic.toml index 49457147..a5015e22 100644 --- a/htsget-config/examples/config-files/basic.toml +++ b/htsget-config/examples/config-files/basic.toml @@ -1,18 +1,6 @@ # An example of running htsget-rs. # Run with `cargo run --all-features -- --config htsget-config/examples/config-files/basic.toml` -ticket_server_addr = "127.0.0.1:8080" -data_server_addr = "127.0.0.1:8081" +locations = "file://data/" -# Serve data locally from the `data` directory. -[[resolvers]] -regex = '.*' -substitution_string = '$0' -storage.backend = 'Local' - -# Serve data from S3 if the id is prefixed with `example_bucket`. -[[resolvers]] -regex = '^(example_bucket)/(?P.*)$' -substitution_string = '$key' -storage.backend = 'S3' # Uses the first capture group in the regex as the bucket. diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index 8caf5b69..508fcdb8 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -2,12 +2,14 @@ //! use crate::config::advanced::regex_location::RegexLocation; +use crate::error::{Error::ParseError, Result}; use crate::storage; use crate::storage::file::default_authority; use crate::storage::Backend; use crate::types::Scheme; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; +use std::result; #[cfg(feature = "url-storage")] use {crate::config::advanced::url::Url, crate::error, http::Uri}; @@ -56,6 +58,24 @@ impl LocationEither { LocationEither::Regex(regex_location) => regex_location.backend(), } } + + /// Get the simple location variant, returning an error otherwise. + pub fn as_simple(&self) -> Result<&Location> { + if let LocationEither::Simple(simple) = self { + Ok(simple) + } else { + Err(ParseError("not a `Simple` variant".to_string())) + } + } + + /// Get the regex location variant, returning an error otherwise. + pub fn as_regex(&self) -> Result<&RegexLocation> { + if let LocationEither::Regex(regex) = self { + Ok(regex) + } else { + Err(ParseError("not a `Regex` variant".to_string())) + } + } } impl Default for LocationEither { @@ -153,7 +173,7 @@ impl From for LocationEither { } impl<'de> Deserialize<'de> for StringLocation { - fn deserialize(deserializer: D) -> Result + fn deserialize(deserializer: D) -> result::Result where D: Deserializer<'de>, { diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 06b71f12..84f9b0f0 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -8,8 +8,9 @@ use crate::config::location::{Location, LocationEither, Locations}; use crate::config::parser::from_path; use crate::config::service_info::ServiceInfo; use crate::config::ticket_server::TicketServerConfig; -use crate::error::Error::{ArgParseError, TracingError}; +use crate::error::Error::{ArgParseError, ParseError, TracingError}; use crate::error::Result; +use crate::storage::file::{default_path, File}; use crate::storage::Backend; use clap::{Args as ClapArgs, Command, FromArgMatches, Parser}; use serde::{Deserialize, Serialize}; @@ -105,10 +106,14 @@ impl Config { /// Parse the command line arguments. Returns the config path, or prints the default config. /// Augment the `Command` args from the `clap` parser. Returns an error if the pub fn parse_args_with_command(augment_args: Command) -> Result> { - Ok(Self::parse_with_args( - Args::from_arg_matches(&Args::augment_args(augment_args).get_matches()) - .map_err(|err| ArgParseError(err.to_string()))?, - )) + let args = Args::from_arg_matches(&Args::augment_args(augment_args).get_matches()) + .map_err(|err| ArgParseError(err.to_string()))?; + + if !args.config.as_ref().is_some_and(|path| path.exists()) { + return Err(ParseError("config file not found".to_string())); + } + + Ok(Self::parse_with_args(args)) } /// Parse the command line arguments. Returns the config path, or prints the default config. @@ -165,17 +170,21 @@ impl Config { .iter_mut() .map(|location| { if let LocationEither::Simple(simple) = location { - // Fall through only if the backend is File. - if simple.backend().as_file().is_err() { + // Fall through only if the backend is File and default + let file_location = if let Ok(location) = simple.backend().as_file() { + location + } else { return Ok(()); - } + }; if let DataServerEnabled::Some(ref data_server) = self.data_server { let prefix = simple.prefix().to_string(); - *location = LocationEither::Simple(Location::new( - Backend::File(data_server.try_into()?), - prefix, - )); + + // Don't update the local path as that comes in from the config. + let file: File = data_server.try_into()?; + let file = file.set_local_path(file_location.local_path().to_string()); + + *location = LocationEither::Simple(Location::new(Backend::File(file), prefix)); } } @@ -488,13 +497,30 @@ pub(crate) mod tests { "#, |config| { assert_eq!(config.locations().len(), 1); - if let LocationEither::Simple(regex) = config.locations.into_inner().first().unwrap() { - println!("{:#?}", regex); - assert!(matches!(regex.backend(), + let config = config.locations.into_inner(); + let regex = config[0].as_regex().unwrap(); + assert!(matches!(regex.backend(), Backend::File(file) if file.local_path() == "path" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); - } else { - panic!(); - } + }, + ); + } + + #[test] + fn simple_locations() { + test_config_from_file( + r#" + data_server.addr = "127.0.0.1:8080" + data_server.local_path = "path" + + locations = "file://data" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), ""); + assert!(matches!(location.backend(), + Backend::File(file) if file.local_path() == "data" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); }, ); } diff --git a/htsget-config/src/storage/file.rs b/htsget-config/src/storage/file.rs index ace26341..2b5ba300 100644 --- a/htsget-config/src/storage/file.rs +++ b/htsget-config/src/storage/file.rs @@ -67,6 +67,12 @@ impl File { pub fn keys(&self) -> Option<&C4GHKeys> { self.keys.as_ref() } + + /// Set the local path. + pub fn set_local_path(mut self, local_path: String) -> Self { + self.local_path = local_path; + self + } } impl Default for File { From 23dd558fd13b1161f42fff5a065c642bbf643819 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 16 Dec 2024 14:13:03 +1100 Subject: [PATCH 08/12] docs(config): describe basic config in docs --- htsget-actix/benches/request_benchmarks.rs | 2 +- htsget-actix/src/lib.rs | 8 +- htsget-axum/src/server/data.rs | 15 +- htsget-axum/src/server/ticket.rs | 8 +- htsget-config/README.md | 82 +++++++++- .../examples/config-files/basic.toml | 3 +- htsget-config/src/config/data_server.rs | 8 +- htsget-config/src/config/mod.rs | 146 +++++++++++++++++- htsget-config/src/resolver.rs | 12 +- htsget-config/src/storage/file.rs | 2 - htsget-storage/src/lib.rs | 3 +- 11 files changed, 248 insertions(+), 41 deletions(-) diff --git a/htsget-actix/benches/request_benchmarks.rs b/htsget-actix/benches/request_benchmarks.rs index d13184b2..4b60d5e9 100644 --- a/htsget-actix/benches/request_benchmarks.rs +++ b/htsget-actix/benches/request_benchmarks.rs @@ -150,7 +150,7 @@ fn start_htsget_rs() -> (DropGuard, String) { let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let data_server = config.data_server().clone().unwrap(); + let data_server = config.data_server().clone().as_data_server_config(); let htsget_rs_ticket_url = format!("http://{}", data_server.addr()); query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); diff --git a/htsget-actix/src/lib.rs b/htsget-actix/src/lib.rs index fd0a1991..02a37c63 100644 --- a/htsget-actix/src/lib.rs +++ b/htsget-actix/src/lib.rs @@ -215,10 +215,14 @@ mod tests { #[async_trait(?Send)] impl TestServer> for ActixTestServer { async fn get_expected_path(&self) -> String { - let data_server = self.get_config().data_server().clone().unwrap(); + let data_server = self + .get_config() + .data_server() + .as_data_server_config() + .unwrap(); let path = data_server.local_path().to_path_buf(); - let mut bind_data_server = BindServer::from(data_server); + let mut bind_data_server = BindServer::from(data_server.clone()); let server = bind_data_server.bind_data_server().await.unwrap(); let addr = server.local_addr(); diff --git a/htsget-axum/src/server/data.rs b/htsget-axum/src/server/data.rs index 15ec8e81..d74d5b9e 100644 --- a/htsget-axum/src/server/data.rs +++ b/htsget-axum/src/server/data.rs @@ -6,7 +6,6 @@ use crate::server::{configure_cors, BindServer, Server}; use axum::Router; use htsget_config::config::advanced::cors::CorsConfig; use htsget_config::config::data_server::DataServerConfig; -use htsget_config::storage::file::PATH_PREFIX; use std::net::SocketAddr; use std::path::Path; use tokio::task::JoinHandle; @@ -35,7 +34,7 @@ impl DataServer { /// Create the router for the data server. pub fn router>(cors: CorsConfig, path: P) -> Router { Router::new() - .nest_service(PATH_PREFIX, ServeDir::new(path)) + .nest_service("/", ServeDir::new(path)) .layer(configure_cors(cors)) .layer(TraceLayer::new_for_http()) } @@ -205,8 +204,9 @@ mod tests { let (_, base_path) = create_local_test_files().await; let data_server = config_with_tls(base_path.path()) .data_server() - .clone() - .unwrap(); + .as_data_server_config() + .unwrap() + .clone(); let server_config = data_server.into_tls().unwrap(); test_server("https", Some(server_config), base_path.path().to_path_buf()).await; @@ -262,9 +262,10 @@ mod tests { let tmp_dir = tempdir().unwrap(); let data_server = config_with_tls(tmp_dir.path()) .data_server() - .clone() - .unwrap(); - let server_config = data_server.into_tls().unwrap(); + .as_data_server_config() + .unwrap() + .clone(); + let server_config = data_server.clone().into_tls().unwrap(); BindServer::new_with_tls( "127.0.0.1:8080".parse().unwrap(), diff --git a/htsget-axum/src/server/ticket.rs b/htsget-axum/src/server/ticket.rs index 4be79d84..211759f1 100644 --- a/htsget-axum/src/server/ticket.rs +++ b/htsget-axum/src/server/ticket.rs @@ -170,10 +170,14 @@ mod tests { #[async_trait(?Send)] impl TestServer>> for AxumTestServer { async fn get_expected_path(&self) -> String { - let data_server = self.get_config().data_server().clone().unwrap(); + let data_server = self + .get_config() + .data_server() + .as_data_server_config() + .unwrap(); let path = data_server.local_path().to_path_buf(); - let mut bind_data_server = BindServer::from(data_server); + let mut bind_data_server = BindServer::from(data_server.clone()); let server = bind_data_server.bind_data_server().await.unwrap(); let addr = server.local_addr(); diff --git a/htsget-config/README.md b/htsget-config/README.md index 6e7d00fd..2b96396b 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -29,14 +29,88 @@ cargo run -p htsget-axum -- --config This will serve files under the [`data`][data] directory: ```sh -curl 'http://localhost:8080/variants/data/vcf/sample1-bcbio-cancer' +curl 'http://localhost:8080/reads/bam/htsnexus_test_NA12878' ``` -Locations allow htsget-rs access to bioinformatics files and indexes. In this example, htsget-rs can read For example, to give htsget-rs access to the data directory, configure -a file location: +Locations allow htsget-rs access to bioinformatics files and indexes. Instead of local files, htsget-rs can access +files on s3, which returns pre-signed URLs for tickets: +```toml +locations = "s3://bucket" +``` + +or on a remote HTTP server: + +```toml +locations = "https://example.com" +``` + +Multiple locations can be specified by providing a list and an id prefix after the location: + +```toml +locations = ["file://data/bam", "file://data/cram"] +``` + +This allows htsget-rs to serve data only when the request also contains the prefix: + +```sh +curl 'http://localhost:8080/reads/bam/htsnexus_test_NA12878' +curl 'http://localhost:8080/reads/cram/htsnexus_test_NA12878?format=CRAM' +``` + +Locations can be mixed, and don't all need to have the same directory or resource: + +```toml +data_server.local_path = "root" +locations = ["file://dir_two/bam", "file://dir_one/cram", "s3://bucket/vcf"] +``` + +htsget-rs spawns a separate server process to respond to htsget tickets for file locations, +so setting `data_server.local_path` to the root directory which contains all subdirectories is +required to give this server access to the local directory. + +The data server process can be disabled by setting it to `None` if no file locations are being used: + +```toml +data_server = "None" +``` + +## Server config + +htsget-rs spawn up to two server instances - the ticket server responds to the initial htsget request, and optionally, +the data server, which responds to the htsget tickets. + +The socket address of the servers can be changed by specifying `addr`: + +```toml +ticket_server.addr = "127.0.0.1:8000" +data_server.addr = "127.0.0.1:8001" +``` + +TLS can be configured to enabled HTTPS support by providing a certificate and private key: + +```toml +ticket_server.tls.key = "key.pem" +ticket_server.tls.cert = "cert.pem" + +data_server.tls.key = "key.pem" +data_server.tls.cert = "cert.pem" +``` + +CORS can also be configured: + +```toml +data_server.cors.allow_credentials = false +data_server.cors.allow_origins = "Mirror" +data_server.cors.allow_headers = "All" +data_server.cors.allow_methods = ['GET', 'POST'] +data_server.cors.max_age = 86400 +data_server.cors.expose_headers = [] +``` + +Use `"Mirror"` to mirror CORS requests, and `"All"` to allow all methods, headers, or origins. -This crate is used to configure htsget-rs using a config file or environment variables. +## Advanced config ## Usage diff --git a/htsget-config/examples/config-files/basic.toml b/htsget-config/examples/config-files/basic.toml index a5015e22..44d4f351 100644 --- a/htsget-config/examples/config-files/basic.toml +++ b/htsget-config/examples/config-files/basic.toml @@ -1,6 +1,5 @@ # An example of running htsget-rs. # Run with `cargo run --all-features -- --config htsget-config/examples/config-files/basic.toml` - -locations = "file://data/" +locations = ["file://data/bam", "file://data/cram"] # Uses the first capture group in the regex as the bucket. diff --git a/htsget-config/src/config/data_server.rs b/htsget-config/src/config/data_server.rs index a6219f0e..e9a39447 100644 --- a/htsget-config/src/config/data_server.rs +++ b/htsget-config/src/config/data_server.rs @@ -2,6 +2,7 @@ //! use crate::config::advanced::cors::CorsConfig; +use crate::error::{Error::ParseError, Result}; use crate::storage::file::{default_localstorage_addr, default_path}; use crate::tls::TlsServerConfig; use serde::{Deserialize, Serialize}; @@ -25,11 +26,12 @@ pub enum DataServerEnabled { } impl DataServerEnabled { - pub fn unwrap(self) -> DataServerConfig { + /// Get the data server config, or an error if `None`. + pub fn as_data_server_config(&self) -> Result<&DataServerConfig> { if let Self::Some(config) = self { - config + Ok(config) } else { - panic!("called `DataServerEnabled::unwrap()` on a `None` value") + Err(ParseError("expected `None` variant".to_string())) } } } diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 84f9b0f0..9b12f524 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -10,7 +10,7 @@ use crate::config::service_info::ServiceInfo; use crate::config::ticket_server::TicketServerConfig; use crate::error::Error::{ArgParseError, ParseError, TracingError}; use crate::error::Result; -use crate::storage::file::{default_path, File}; +use crate::storage::file::File; use crate::storage::Backend; use clap::{Args as ClapArgs, Command, FromArgMatches, Parser}; use serde::{Deserialize, Serialize}; @@ -324,7 +324,12 @@ pub(crate) mod tests { vec![("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8082")], |config| { assert_eq!( - config.data_server().clone().unwrap().addr(), + config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .addr(), "127.0.0.1:8082".parse().unwrap() ); }, @@ -359,7 +364,12 @@ pub(crate) mod tests { fn config_data_server_addr_file() { test_config_from_file(r#"data_server.addr = "127.0.0.1:8082""#, |config| { assert_eq!( - config.data_server().clone().unwrap().addr(), + config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .addr(), "127.0.0.1:8082".parse().unwrap() ); }); @@ -379,7 +389,13 @@ pub(crate) mod tests { key_path.to_string_lossy().escape_default() ), |config| { - assert!(config.data_server().clone().unwrap().tls().is_none()); + assert!(config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .tls() + .is_none()); }, ); }); @@ -401,7 +417,13 @@ pub(crate) mod tests { cert_path.to_string_lossy().escape_default() ), |config| { - assert!(config.data_server().clone().unwrap().tls().is_some()); + assert!(config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .tls() + .is_some()); }, ); }); @@ -419,7 +441,13 @@ pub(crate) mod tests { ("HTSGET_DATA_SERVER_TLS_CERT", cert_path.to_string_lossy()), ], |config| { - assert!(config.data_server().clone().unwrap().tls().is_some()); + assert!(config + .data_server() + .clone() + .as_data_server_config() + .unwrap() + .tls() + .is_some()); }, ); }); @@ -519,9 +547,113 @@ pub(crate) mod tests { let config = config.locations.into_inner(); let location = config[0].as_simple().unwrap(); assert_eq!(location.prefix(), ""); + assert_file_location(location, "data"); + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn simple_locations_s3() { + test_config_from_file( + r#" + locations = "s3://bucket" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), ""); + assert!(matches!(location.backend(), + Backend::S3(s3) if s3.bucket() == "bucket")); + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn simple_locations_url() { + test_config_from_file( + r#" + locations = "https://example.com" + "#, + |config| { + assert_eq!(config.locations().len(), 1); + let config = config.locations.into_inner(); + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), ""); + assert!(matches!(location.backend(), + Backend::Url(url) if url.url() == &"https://example.com".parse::().unwrap())); + }, + ); + } + + #[test] + fn simple_locations_multiple() { + test_config_from_file( + r#" + locations = ["file://data/bam", "file://data/cram"] + "#, + |config| { + assert_eq!(config.locations().len(), 2); + let config = config.locations.into_inner(); + + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), "bam"); + assert_file_location(location, "data"); + + let location = config[1].as_simple().unwrap(); + assert_eq!(location.prefix(), "cram"); + assert_file_location(location, "data"); + }, + ); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn simple_locations_multiple_mixed() { + test_config_from_file( + r#" + data_server.local_path = "root" + locations = ["file://dir_one/bam", "file://dir_two/cram", "s3://bucket/vcf"] + "#, + |config| { + assert_eq!(config.locations().len(), 2); + let config = config.locations.into_inner(); + + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), "bam"); + assert_file_location(location, "dir_one"); + + let location = config[1].as_simple().unwrap(); + assert_eq!(location.prefix(), "cram"); + assert_file_location(location, "dir_two"); + + let location = config[2].as_simple().unwrap(); + assert_eq!(location.prefix(), "vcf"); assert!(matches!(location.backend(), - Backend::File(file) if file.local_path() == "data" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); + Backend::S3(s3) if s3.bucket() == "bucket")); + }, + ); + } + + #[test] + fn no_data_server() { + test_config_from_file( + r#" + data_server = "None" + "#, + |config| { + assert!(matches!( + config.data_server().as_data_server_config(), + Err(_) + )); }, ); } + + fn assert_file_location(location: &Location, local_path: &str) { + assert!(matches!(location.backend(), + Backend::File(file) if file.local_path() == local_path && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); + } } diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index 82c39726..28731bf9 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -4,7 +4,6 @@ use crate::storage; use crate::storage::{Backend, ResolvedId}; use crate::types::{Query, Response, Result}; use async_trait::async_trait; -use std::path::Path; use tracing::instrument; /// A trait which matches the query id, replacing the match in the substitution text. @@ -86,12 +85,7 @@ impl IdResolver for LocationEither { match self { LocationEither::Simple(location) => { if query.id().starts_with(location.prefix()) { - return Some(ResolvedId::new( - Path::new(location.prefix()) - .join(query.id()) - .to_str()? - .to_string(), - )); + return Some(ResolvedId::new(query.id().to_string())); } } LocationEither::Regex(regex_location) => { @@ -383,7 +377,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-1", Bam)) .unwrap() .into_inner(), - "id-1/id-1" + "id-1" ); assert_eq!( resolver @@ -391,7 +385,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-2", Bam)) .unwrap() .into_inner(), - "id-2/id-2" + "id-2" ); } diff --git a/htsget-config/src/storage/file.rs b/htsget-config/src/storage/file.rs index 2b5ba300..026aa888 100644 --- a/htsget-config/src/storage/file.rs +++ b/htsget-config/src/storage/file.rs @@ -13,8 +13,6 @@ use http::uri::Authority; use serde::{Deserialize, Serialize}; use std::str::FromStr; -pub const PATH_PREFIX: &str = "/data"; - /// Local file based storage. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default, deny_unknown_fields)] diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index 0bc5fac6..fac2351f 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -24,7 +24,6 @@ use cfg_if::cfg_if; use htsget_config::storage; #[cfg(feature = "experimental")] use htsget_config::storage::c4gh::C4GHKeys; -use htsget_config::storage::file::PATH_PREFIX; use htsget_config::types::Scheme; use http::uri; use pin_project_lite::pin_project; @@ -278,7 +277,7 @@ pub trait UrlFormatter { impl UrlFormatter for storage::file::File { fn format_url>(&self, key: K) -> Result { - let path = Path::new("/").join(PATH_PREFIX).join(key.as_ref()); + let path = Path::new("/").join(self.local_path()).join(key.as_ref()); uri::Builder::new() .scheme(match self.scheme() { Scheme::Http => uri::Scheme::HTTP, From d3aab3652dd19f3ebad2e8c3a48571aba57cd131 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 17 Dec 2024 11:25:42 +1100 Subject: [PATCH 09/12] docs(config): add remaining sections in docs refactor --- htsget-config/README.md | 597 ++++++------------ .../src/config/advanced/regex_location.rs | 8 +- htsget-config/src/config/mod.rs | 1 + htsget-config/src/config/service_info.rs | 2 +- htsget-config/src/storage/c4gh/local.rs | 15 +- htsget-config/src/storage/c4gh/mod.rs | 2 +- .../src/storage/c4gh/secrets_manager.rs | 14 +- htsget-config/src/storage/mod.rs | 5 +- 8 files changed, 216 insertions(+), 428 deletions(-) diff --git a/htsget-config/README.md b/htsget-config/README.md index 2b96396b..1dd5c0dc 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -8,12 +8,13 @@ [actions-badge]: https://github.com/umccr/htsget-rs/actions/workflows/action.yml/badge.svg [actions-url]: https://github.com/umccr/htsget-rs/actions?query=workflow%3Atests+branch%3Amain +## Overview + Configuration for [htsget-rs]. [htsget-rs]: https://github.com/umccr/htsget-rs -## Overview - +## Quickstart The simplest way to use htsget-rs is to create a [toml] config file and specify a storage location: ```toml @@ -75,7 +76,7 @@ The data server process can be disabled by setting it to `None` if no file locat data_server = "None" ``` -## Server config +### Server config htsget-rs spawn up to two server instances - the ticket server responds to the initial htsget request, and optionally, the data server, which responds to the htsget tickets. @@ -97,149 +98,45 @@ data_server.tls.key = "key.pem" data_server.tls.cert = "cert.pem" ``` -CORS can also be configured: - -```toml -data_server.cors.allow_credentials = false -data_server.cors.allow_origins = "Mirror" -data_server.cors.allow_headers = "All" -data_server.cors.allow_methods = ['GET', 'POST'] -data_server.cors.max_age = 86400 -data_server.cors.expose_headers = [] -``` - -Use `"Mirror"` to mirror CORS requests, and `"All"` to allow all methods, headers, or origins. - -## Advanced config - -## Usage - -To configure htsget-rs, a TOML config file can be defined. There is also support for reading config from environment variables. -Any config options set by environment variables override values in the config file. - -The configuration consists of TOML tables, such as config for the ticket server, data server, service-info, or resolvers. - -As a starting point, see the [basic TOML][basic] example file which should work for many use-cases. +### Service info config -#### Ticket server config - -The ticket server responds to htsget requests by returning a set of URL tickets that the client must fetch and concatenate. -To configure the ticket server, set the following options: - -| Option | Description | Type | Default | -|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|-----------------------------| -| `ticket_server_addr` | The address for the ticket server. | Socket address | `'127.0.0.1:8080'` | -| `ticket_server_tls` | Enable TLS for the ticket server. See [TLS](#tls) for more details. | TOML table | Not enabled | -| `ticket_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the ticket server. | Boolean | `false` | -| `ticket_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the ticket server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | -| `ticket_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the ticket server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | -| `ticket_server_cors_allow_methods` | Set the CORS Access-Control-Allow-Methods returned by the ticket server, this can be set to `All` to allow all methods, or a specific array of methods. | `'All'`, or a array of methods | `'All'` | -| `ticket_server_cors_max_age` | Set the CORS Access-Control-Max-Age for the ticket server which controls how long a preflight request can be cached for. | Seconds | `86400` | -| `ticket_server_cors_expose_headers` | Set the CORS Access-Control-Expose-Headers returned by the ticket server, this can be set to `All` to expose all headers, or a specific array of headers. | `'All'`, or a array of headers | `[]` | - -TLS is supported by setting the `ticket_server_key` and `ticket_server_cert` options. An example of config for the ticket server: -```toml -ticket_server_addr = '127.0.0.1:8080' -ticket_server_cors_allow_credentials = false -ticket_server_cors_allow_origins = 'Mirror' -ticket_server_cors_allow_headers = ['Content-Type'] -ticket_server_cors_allow_methods = ['GET', 'POST'] -ticket_server_cors_max_age = 86400 -ticket_server_cors_expose_headers = [] -``` +The service info config controls what is returned when the [`service-info`][service-info] path is queried. The following +option accepts any nested value, which gets converted to a JSON response: -#### Data server config - -The local data server responds to tickets produced by the ticket server by serving local filesystem data. -To configure the data server, set the following options: - -| Option | Description | Type | Default | -|-------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------|-----------------------------| -| `data_server_addr` | The address for the data server. | Socket address | `'127.0.0.1:8081'` | -| `data_server_local_path` | The local path which the data server can access to serve files. | Filesystem path | `'./'` | -| `data_server_serve_at` | The path which the data server will prefix to all response URLs for tickets. | URL path | `''` | -| `data_server_tls` | Enable TLS for the data server. See [TLS](#tls) for more details. | TOML table | Not enabled | -| `data_server_cors_allow_credentials` | Controls the CORS Access-Control-Allow-Credentials for the data server. | Boolean | `false` | -| `data_server_cors_allow_origins` | Set the CORS Access-Control-Allow-Origin returned by the data server, this can be set to `All` to send a wildcard, `Mirror` to echo back the request sent by the client, or a specific array of origins. | `'All'`, `'Mirror'` or a array of origins | `['http://localhost:8080']` | -| `data_server_cors_allow_headers` | Set the CORS Access-Control-Allow-Headers returned by the data server, this can be set to `All` to allow all headers, or a specific array of headers. | `'All'`, or a array of headers | `'All'` | -| `data_server_cors_allow_methods` | Set the CORS Access-Control-Allow-Methods returned by the data server, this can be set to `All` to allow all methods, or a specific array of methods. | `'All'`, or a array of methods | `'All'` | -| `data_server_cors_max_age` | Set the CORS Access-Control-Max-Age for the data server which controls how long a preflight request can be cached for. | Seconds | `86400` | -| `data_server_cors_expose_headers` | Set the CORS Access-Control-Expose-Headers returned by the data server, this can be set to `All` to expose all headers, or a specific array of headers. | `'All'`, or a array of headers | `[]` | - -TLS is supported by setting the `data_server_key` and `data_server_cert` options. An example of config for the data server: ```toml -data_server_addr = '127.0.0.1:8081' -data_server_local_path = './' -data_server_serve_at = '' -data_server_key = 'key.pem' -data_server_cert = 'cert.pem' -data_server_cors_allow_credentials = false -data_server_cors_allow_origins = 'Mirror' -data_server_cors_allow_headers = ['Content-Type'] -data_server_cors_allow_methods = ['GET', 'POST'] -data_server_cors_max_age = 86400 -data_server_cors_expose_headers = [] +service_info.environment = "dev" +service_info.organization = { name = "name", url = "https://example.com/" } ``` -Sometimes it may be useful to disable the data server as all responses to the ticket server will be handled elsewhere, such as with an AWS S3 data server. - -To disable the data server, set the following option: - -
-data_server_enabled = false
-
- -#### Service info config - -The service info config controls what is returned when the [`service-info`][service-info] path is queried.
-To configure the service-info, set the following options: - -| Option | Description | Type | Default | -|---------------------------------------------------------|---------------------------------------------|-----------|----------| -| `id` | Service ID. | String | Not set | -| `name` | Service name. | String | Not set | -| `version` | Service version. | String | Not set | -| `organization_name` | Organization name. | String | Not set | -| `organization_url` | Organization URL. | String | Not set | -| `contact_url` | Service contact URL | String | Not set | -| `documentation_url` | Service documentation URL. | String | Not set | -| `created_at` | When the service was created. | String | Not set | -| `updated_at` | When the service was last updated. | String | Not set | -| `environment` | The environment the service is running in. | String | Not set | - -An example of config for the service info: -```toml -id = 'id' -name = 'name' -version = '0.1' -organization_name = 'name' -organization_url = 'https://example.com/' -contact_url = 'mailto:nobody@example.com' -documentation_url = 'https://example.com/' -created_at = '2022-01-01T12:00:00Z' -updated_at = '2022-01-01T12:00:00Z' -environment = 'dev' -``` +### Environment variables -#### Resolvers +Most options can also be set using environment variables. Any environment variables will override options set in the +config file. Arrays are delimited with `[` and `]`, and items are separated by commas: -The resolvers component of htsget-rs is used to map query IDs to the location of the resource. This is the component of the -code that takes the [`id`][id], which is everything after `reads/` or `variants/` in the http path, and maps it to a data location. +| Variable | Description | Example | +|---------------------------------|----------------------------------------------------------------|----------------------------------------------------| +| `HTSGET_TICKET_SERVER_ADDR` | Set the ticket server socket address. | "127.0.0.1:8080" | +| `HTSGET_TICKET_SERVER_TLS_KEY` | See [server config](#server-config) | "key.pem" | +| `HTSGET_TICKET_SERVER_TLS_CERT` | See [server config](#server-config) | "cert.pem" | +| `HTSGET_DATA_SERVER_ADDR` | Set the data server socket address. | "127.0.0.1:8081" | +| `HTSGET_DATA_SERVER_LOCAL_PATH` | Set the path that the data server has access to. | "dir/path" | +| `HTSGET_DATA_SERVER_TLS_KEY` | See [server config](#server-config) | "key.pem" | +| `HTSGET_DATA_SERVER_TLS_CERT` | See `server config](#server-config) | "cert.pem" | +| `HTSGET_SERVICE_INFO` | Set the service info, see [service info](#service-info-config) | "{ organization = { name = name, url = url }}" | +| `HTSGET_LOCATIONS` | Set the locations. | "[file://data/prefix_one, s3://bucket/prefix_two]" | +| `HTSGET_CONFIG` | Set the config file location. | "dir/config.toml" | -For example, if the request to htsget-rs is: +## Advanced config -```sh -curl 'http://localhost:8080/reads/some_id/file' -``` +The following section describes advanced configuration which is more flexible, but adds complexity. -Then the resolvers controls how the server finds `some_id/file`, which may be stored locally, in the cloud, or at an arbitrary URL location. -The resolvers maps `some_id/file` to a location using regexes and substitution strings. The location of the file does not -need to have the same name as the id. +### Regex-based location -A query ID is matched with a regex, and is then mapped with a substitution string that has access to the regex capture groups. -Resolvers are configured in an array, where the first matching resolver is resolver used to map the ID. +Instead of the simple path-based locations described above, htsget-rs supports arbitrary regex-based id resolution. +This allows matching an [`id`][id] using, which is everything after `reads/` or `variants/` in the http path, and mapping +it to a location using regex substitution. -To create a resolver, add a `[[resolvers]]` array of tables, and set the following options: +To create a regex location, add a `[[locations]]` array of tables, and set the following options: | Option | Description | Type | Default | |-----------------------|-------------------------------------------------------------------------------------------------------------------------|---------------------------------------|---------| @@ -247,7 +144,7 @@ To create a resolver, add a `[[resolvers]]` array of tables, and set the followi | `substitution_string` | The replacement expression used to map the matched query ID. This has access to the match groups in the `regex` option. | String with access to capture groups | `'$0'` | For example, below is a `regex` option which matches a `/` between two groups, and inserts an additional `data` -in between the groups with the `substitution_string`. +in between the groups with the `substitution_string`: ```toml [[resolvers]] @@ -257,96 +154,100 @@ substitution_string = '$group1/data/$group2' This would mean that a request to `http://localhost:8080/reads/some_id/file` would search for files at `some_id/data/file.bam` and `some_id/data/file.bam.bai`. -For more information about regex options see the [regex crate](https://docs.rs/regex/). +The regex locations also have access to further configuration of storage locations for `file://`, `s3://`, or `http://` +locations. These are called `File`, `S3`, and `Url` respectively. -Each resolver also maps to a certain storage backend. This storage backend can be used to set query IDs which are served from local storage, from S3-style bucket storage, or from HTTP URLs. -To set the storage backend for a resolver, add a `[resolvers.storage]` table. Some storage backends require feature flags to be set when compiling htsget-rs. - -To use `LocalStorage`, set `backend = 'Local'` under `[resolvers.storage]`, and specify any additional options from below: +To manually configure `File` locations, set `backend.kind = "File"`, and specify any additional options from below the `backend` table: | Option | Description | Type | Default | |--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| | `scheme` | The scheme present on URL tickets. | Either `'Http'` or `'Https'` | `'Http'` | -| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | URL authority | `'127.0.0.1:8081'` | -| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Filesystem path | `'./'` | -| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `''` | -| `use_data_server_config` | Whether to use the data server config to fill in the above values. This overrides any other options specified from this table. | Boolean | `false` | +| `authority` | The authority present on URL tickets. This should likely match the `data_server.addr`. | URL authority | `'127.0.0.1:8081'` | +| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server.local_path`. | Filesystem path | `'./'` | -By default, if the above options are left unspecified, they inherit values from the [`data_server`][data-server] config. -For example, the following sets the `scheme`, `authority`, `local_path` and `path_prefix` to values used by the `data_server`. +For example: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' +[[locations]] +regex = ".*" +substitution_string = "$0" -[resolvers.storage] -backend = 'Local' +backend.kind = "Local" +backend.scheme = "Http" +backend.authority = "127.0.0.1:8000" +backend.local_path = "path" ``` -To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `backend = 'S3'` under `[resolvers.storage]`, and specify: +To manually configure `S3` locations, set `backend.kind = "S3"`, and specify options from below under the `backend` table: + +| Option | Description | Type | Default | +|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------| +| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `location` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. | +| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. | +| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | -| Option | Description | Type | Default | -|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------| -| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `resolvers` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. | -| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. | -| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | +For example, the following backend manually sets the `bucket` and uses path style requests: -For example, a `resolvers` value of: ```toml -[[resolvers]] -regex = '^(example_bucket)/(?P.*)$' -substitution_string = '$key' +[[locations]] +regex = "prefix/(?P.*)$" +substitution_string = "$key" -[resolvers.storage] -backend = 'S3' -# Uses the first capture group in the regex as the bucket. +backend.kind = "S3" +backend.bucket = "bucket" +backend.path_style = true ``` -Will use "example_bucket" as the S3 bucket if that resolver matches, because this is the first capture group in the `regex`. -Note, to use this feature, at least one capture group must be defined in the `regex`. +To manually configure `Url` locations, set `backend.kind = "Url"`, specify any additional options from below under the `backend` table: -`UrlStorage` is a storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. -To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `backend = 'Url'` under `[resolvers.storage]`, and specify any additional options from below: +| Option | Description | Type | Default | +|--------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------| +| `url` | The URL to fetch data from. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `response_url` | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. | Boolean | `true` | +| `header_blacklist` | List of headers that should not be forwarded. | Array of headers | `[]` | +| `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [server configuration](#server-configuration) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. | -| Option | Description | Type | Default | -|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------| -| `url` | The URL to fetch data from. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `response_url` | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. | Boolean | `true` | -| `header_blacklist` | List of headers that should not be forwarded. | Array of headers | `[]` | -| `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [TLS](#tls) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. | +For example, the following forwards all headers to response tickets except `Host`, and constructs tickets using `https://example.com` instead of `http://localhost:8080`: -When using `UrlStorage`, the following requests will be made to the `url`. -* `GET` request to fetch only the headers of the data file (e.g. `GET /data.bam`, with `Range: bytes=0-`). -* `GET` request to fetch the entire index file (e.g. `GET /data.bam.bai`). -* `HEAD` request on the data file to get its length (e.g. `HEAD /data.bam`). +```toml +[[locations]] +regex = ".*" +substitution_string = "$0" -By default, all headers received in the initial query will be included when making these requests. To exclude certain headers from being forwarded, set the `header_blacklist` option. Note that the blacklisted headers are removed from the requests made to `url` and from the URL tickets as well. +backend.kind = "Url" +backend.url = "http://localhost:8080" +backend.response_url = "https://example.com" +backend.forward_headers = true +backend.header_blacklist = ["Host"] +``` -Example of a resolver with `UrlStorage`: +Regex-based locations also support multiple locations: ```toml -[[resolvers]] +[[locations]] +regex = "prefix/(?P.*)$" +substitution_string = "$key" +backend.kind = "S3" +backend.bucket = "bucket" +backend.path_style = true + +[[locations]] regex = ".*" substitution_string = "$0" - -[resolvers.storage] -backend = 'Url' -url = "http://localhost:8080" -response_url = "https://example.com" -forward_headers = true -header_blacklist = ["Host"] +backend.kind = "Url" +backend.url = "http://localhost:8080" +forward_headers = false ``` -There are additional examples of config files located under [`examples/config-files`][examples-config-files]. +If there is an overlap in regex matches, the first location specified will be the one used. -#### Allow guard -Additionally, the resolver component has a feature, which allows resolving IDs based on the other fields present in a query. -This is useful as it allows the resolver to match an ID only if a particular set of query parameters are also present. For example, -a resolver can be set to only resolve IDs if the format is also BAM. +### Allow guard -This component can be configured by setting the `[resolver.allow_guard]` table with. The following options are available to restrict which queries are resolved by a resolver: +Additionally, locations support resolving IDs based on the other fields present in a query. +This is useful to allow the location to match an ID only if a particular set of query parameters are also present. + +This component can be configured by setting the `guard` table with: | Option | Description | Type | Default | |-------------------------|-----------------------------------------------------------------------------------------|-----------------------------------------------------------------------|-------------------------------------| @@ -356,51 +257,34 @@ This component can be configured by setting the `[resolver.allow_guard]` table w | `allow_formats` | Resolve the query ID if the query is one of the formats specified by this option. | An array of formats containing `'BAM'`, `'CRAM'`, `'VCF'`, or `'BCF'` | `['BAM', 'CRAM', 'VCF', 'BCF']` | | `allow_classes` | Resolve the query ID if the query is one of the classes specified by this option. | An array of classes containing eithr `'body'` or `'header'` | `['body', 'header']` | | `allow_interval_start` | Resolve the query ID if the query reference start position is at least this option. | Unsigned 32-bit integer start position, 0-based, inclusive | Not set, allows all start positions | -| `allow_interval_end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive. | Not set, allows all end positions | +| `allow_interval_end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive | Not set, allows all end positions | -An example of a fully configured resolver: +For example, match only if the request queries `chr1` with positions between `100` and `1000`: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' - -[resolvers.storage] -backend = 'S3' -bucket = 'bucket' - -[resolvers.allow_guard] -allow_reference_names = ['chr1'] -allow_fields = ['QNAME'] -allow_tags = ['RG'] -allow_formats = ['BAM'] -allow_classes = ['body'] -allow_interval_start = 100 -allow_interval_end = 1000 -``` - -In this example, the resolver will only match the query ID if the query is for `chr1` with positions between `100` and `1000`. +[[locations]] +regex = ".*" +substitution_string = "$0" -#### TLS +backend.kind = "S3" +backend.bucket = "bucket" -TLS can be configured for the ticket server, data server, or the url storage client. These options read private keys and -certificates from PEM-formatted files. Certificates must be in X.509 format and private keys can be RSA, PKCS8, or SEC1 (EC) encoded. -The following options are available: +guard.allow_reference_names = ["chr1"] +guard.allow_interval_start = 100 +guard.allow_interval_end = 1000 +``` -| Option | Description | Type | Default | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| -| `key` | The path to the PEM formatted X.509 certificate. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | -| `cert` | The path to the PEM formatted RSA, PKCS8, or SEC1 encoded EC private key. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | -| `root_store` | The path to the PEM formatted root certificate store. Only used to specify non-native root certificates for the HTTP client in `UrlStorage`. | Filesystem path | Not Set | +### Server configuration -When used by the ticket and data servers, `key` and `cert` enable TLS, and when used with the url storage client, they enable client authentication. -The root store is only used by the url storage client. Note, the url storage client always allows TLS, however the default configuration performs no client authentication -and uses the native root certificate store. +To use custom root certificates for `Url` locations, set the following: -For example, TLS for the ticket server can be enabled by specifying the key and cert options: ```toml -ticket_server_tls.cert = "cert.pem" -ticket_server_tls.key = "key.pem" +[[locations]] +regex = ".*" +substitution_string = "$0" + +backend.kind = "Url" +backend.tls.root_store = "root.crt" ``` This project uses [rustls] for all TLS logic, and it does not depend on OpenSSL. The rustls library can be more @@ -423,166 +307,36 @@ openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -days 365 -out ser # certificate if using client authentication. ``` -The `root.crt` can then be used in htsget-rs to allow authenticating to a `UrlStorage` backend using `server.crt`: +CORS can also be configured for the data and ticket servers by specifying the `cors` option: ```toml -# Trust the root CA that signed the server's certificate. -tls.root_store = "root.crt" -``` - -Alternatively, projects such as [mkcert] can be used to simplify this process. - -Further TLS examples are available under [`examples/config-files`][examples-config-files]. - -[examples-config-files]: examples/config-files -[rustls]: https://github.com/rustls/rustls -[mkcert]: https://github.com/FiloSottile/mkcert - -#### Config file location - -The htsget-rs binaries ([htsget-axum], [htsget-actix] and [htsget-lambda]) support some command line options. The config file location can -be specified by setting the `--config` option: - -```shell -cargo run -p htsget-axum -- --config "config.toml" -``` - -The config can also be read from an environment variable: - -```shell -export HTSGET_CONFIG="config.toml" -``` - -If no config file is specified, the default configuration is used. Further, the default configuration file can be printed to stdout by passing -the `--print-default-config` flag: - -```shell -cargo run -p htsget-axum -- --print-default-config -``` - -Use the `--help` flag to see more details on command line options. - -[htsget-actix]: ../htsget-actix -[htsget-axum]: ../htsget-axum -[htsget-lambda]: ../htsget-lambda - -#### Log formatting - -The [Tracing][tracing] crate is used by htsget-rs is for logging functionality. The `RUST_LOG` variable is -read to configure the level that trace logs are emitted. - -For example, the following indicates trace level for all htsget crates, and info level for all other crates: - -```sh -export RUST_LOG='info,htsget_lambda=trace,htsget_lambda=trace,htsget_config=trace,htsget_http=trace,htsget_search=trace,htsget_test=trace' -``` - -See [here][rust-log] for more information on setting this variable. - -The style of formatting can be configured by setting the following option: - -| Option | Description | Type | Default | -|---------------------------------------------------------|--------------------------------------|--------------------------------------------------------|----------| -| `formatting_style` | The style of log formatting to use. | One of `'Full'`, `'Compact'`, `'Pretty'`, or `'Json'` | `'Full'` | - -See [here][formatting-style] for more information on how these values look. - -[tracing]: https://github.com/tokio-rs/tracing -[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html -[formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters - -#### Environment variables - -All the htsget-rs config options can be set using environment variables, which is convenient for runtimes such as AWS Lambda. -The ticket server, data server and service info options are flattened and can be set directly using -environment variable. It is not recommended to set the resolvers using environment variables, however it can be done by setting a single environment variable which -contains a list of structures, where a key name and value pair is used to set the nested options. - -Environment variables will override options set in the config file. Note, arrays are delimited with `[` and `]` in environment variables, and items are separated by commas. - -The following environment variables - corresponding to the TOML config - are available: - -| Variable | Description | -|-----------------------------------------------|-------------------------------------------------------------------------------------| -| `HTSGET_TICKET_SERVER_ADDR` | See [`ticket_server_addr`](#ticket_server_addr) | -| `HTSGET_TICKET_SERVER_TLS_KEY` | See [`TLS`](#tls) | -| `HTSGET_TICKET_SERVER_TLS_CERT` | See [`TLS`](#tls) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_CREDENTIALS` | See [`ticket_server_cors_allow_credentials`](#ticket_server_cors_allow_credentials) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS` | See [`ticket_server_cors_allow_origins`](#ticket_server_cors_allow_origins) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_HEADERS` | See [`ticket_server_cors_allow_headers`](#ticket_server_cors_allow_headers) | -| `HTSGET_TICKET_SERVER_CORS_ALLOW_METHODS` | See [`ticket_server_cors_allow_methods`](#ticket_server_cors_allow_methods) | -| `HTSGET_TICKET_SERVER_CORS_MAX_AGE` | See [`ticket_server_cors_max_age`](#ticket_server_cors_max_age) | -| `HTSGET_TICKET_SERVER_CORS_EXPOSE_HEADERS` | See [`ticket_server_cors_expose_headers`](#ticket_server_cors_expose_headers) | -| `HTSGET_DATA_SERVER_ADDR` | See [`data_server_addr`](#data_server_addr) | -| `HTSGET_DATA_SERVER_LOCAL_PATH` | See [`data_server_local_path`](#data_server_local_path) | -| `HTSGET_DATA_SERVER_SERVE_AT` | See [`data_server_serve_at`](#data_server_serve_at) | -| `HTSGET_DATA_SERVER_TLS_KEY` | See [`TLS`](#tls) | -| `HTSGET_DATA_SERVER_TLS_CERT` | See [`TLS`](#tls) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_CREDENTIALS` | See [`data_server_cors_allow_credentials`](#data_server_cors_allow_credentials) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_ORIGINS` | See [`data_server_cors_allow_origins`](#data_server_cors_allow_origins) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_HEADERS` | See [`data_server_cors_allow_headers`](#data_server_cors_allow_headers) | -| `HTSGET_DATA_SERVER_CORS_ALLOW_METHODS` | See [`data_server_cors_allow_methods`](#data_server_cors_allow_methods) | -| `HTSGET_DATA_SERVER_CORS_MAX_AGE` | See [`data_server_cors_max_age`](#data_server_cors_max_age) | -| `HTSGET_DATA_SERVER_CORS_EXPOSE_HEADERS` | See [`data_server_cors_expose_headers`](#data_server_cors_expose_headers) | -| `HTSGET_ID` | See [`id`](#id) | -| `HTSGET_NAME` | See [`name`](#name) | -| `HTSGET_VERSION` | See [`version`](#version) | -| `HTSGET_ORGANIZATION_NAME` | See [`organization_name`](#organization_name) | -| `HTSGET_ORGANIZATION_URL` | See [`organization_url`](#organization_url) | -| `HTSGET_CONTACT_URL` | See [`contact_url`](#contact_url) | -| `HTSGET_DOCUMENTATION_URL` | See [`documentation_url`](#documentation_url) | -| `HTSGET_CREATED_AT` | See [`created_at`](#created_at) | -| `HTSGET_UPDATED_AT` | See [`updated_at`](#updated_at) | -| `HTSGET_ENVIRONMENT` | See [`environment`](#environment) | -| `HTSGET_RESOLVERS` | See [`resolvers`](#resolvers) | -| `HTSGET_FORMATTING_STYLE` | See [`formatting_style`](#formatting_style) | - -In order to use `HTSGET_RESOLVERS`, the entire resolver config array must be set. The nested array of resolvers structure can be set using name key and value pairs, for example: - -```shell -export HTSGET_RESOLVERS="[{ - regex=regex, - substitution_string=substitution_string, - storage={ - type=S3, - bucket=bucket - }, - allow_guard={ - allow_reference_names=[chr1], - allow_fields=[QNAME], - allow_tags=[RG], - allow_formats=[BAM], - allow_classes=[body], - allow_interval_start=100, - allow_interval_end=1000 - } -}]" +ticket_server.cors.allow_credentials = false +ticket_server.cors.allow_origins = "Mirror" +ticket_server.cors.allow_headers = "All" +ticket_server.cors.allow_methods = ['GET', 'POST'] +ticket_server.cors.max_age = 86400 +ticket_server.cors.expose_headers = [] ``` -Similar to the [data_server](#data_server) option, the data server can be disabled by setting the equivalent environment variable: - -```shell -export HTSGET_DATA_SERVER_ENABLED=false -``` -[service-info]: https://samtools.github.io/hts-specs/htsget.html#ga4gh-service-info +Use `"Mirror"` to mirror CORS requests, and `"All"` to allow all methods, headers, or origins. The `ticket_server` table +above can be replaced with `data_server` to configure CORS for the data server. ### MinIO -Operating a local object storage like [MinIO][minio] can be achieved by leveraging the `endpoint` directive as shown below: +Operating a local object storage like [MinIO][minio] can be achieved by using `endpoint` under `"S3"` locations as shown below: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' - -[resolvers.storage] -backend = 'S3' -bucket = 'bucket' -endpoint = 'http://127.0.0.1:9000' -path_style = true +[[locations]] +regex = ".*" +substitution_string = "$0" + +backend.kind = 'S3' +backend.bucket = 'bucket' +backend.endpoint = 'http://127.0.0.1:9000' +backend.path_style = true ``` -Care must be taken to ensure that the [correct][env-variables] `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY` is set to allow +Care must be taken to ensure that the [correct][env-variables] `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY` are set to allow the AWS sdk to reach the endpoint. Additional configuration of the MinIO server is required to use [virtual-hosted][virtual-addressing] style addressing by setting the `MINIO_DOMAIN` environment variable. [Path][path-addressing] style addressing can be forced using `path_style = true`. @@ -590,54 +344,78 @@ See the MinIO deployment [example][minio-deployment] for more information on how ### Crypt4GH -There is experimental support for serving [Crypt4GH][c4gh] encrypted files. This can be enabled by compiling with the -`experimental` feature flag. +There is experimental support for serving [Crypt4GH][c4gh] encrypted files. This allows htsget-rs to read Crypt4GH files and serve them encrypted, directly to the client. In the process of serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and re-encrypt them so that the client can read them. When the client receives byte ranges from htsget-rs and concatenates them, the output bytes will be Crypt4GH encrypted, and will need to be decrypted before they can be read. All file formats (BAM, CRAM, VCF, and BCF) are supported using Crypt4GH. -To use this feature, set `location = 'Local'` under `resolvers.storage.keys` to specify the private and public keys: +To use this feature, set `keys.kind = "File"` under the `location` table to specify the private and public keys: -| Option | Description | Type | Default | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| -| `private_key` | The path to PEM formatted private key which htsget-rs uses to decrypt Crypt4GH data. | Filesystem path | Not Set | -| `recipient_public_key` | The path to the PEM formatted public key which the recipient of the data will use. This is what the client will use to decrypt the returned data, using the corresponding private key. | Filesystem path | Not Set | +| Option | Description | Type | Default | +|-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| +| `private` | The path to PEM formatted private key which htsget-rs uses to decrypt Crypt4GH data. | Filesystem path | Not Set | +| `public` | The path to the PEM formatted public key which the recipient of the data will use. This is what the client will use to decrypt the returned data, using the corresponding private key. | Filesystem path | Not Set | For example: ```toml [[resolvers]] -regex = '.*' -substitution_string = '$0' +regex = ".*" +substitution_string = "$0" -[resolvers.storage.keys] -location = 'Local' -private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret -recipient_public_key = 'data/c4gh/keys/alice.pub' +location.keys.kind = "File" +location.keys.private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret +location.keys.public = "data/c4gh/keys/alice.pub" ``` -Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `location = 'SecretsManager'` under -`resolvers.storage.keys` to fetch keys from Secrets Manager. When using Secrets Manager, the `private_key` and `recipient_public_key` +Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `keys.kind = "SecretsManager"` under +`location` to fetch keys from Secrets Manager. When using Secrets Manager, the `private` and `public` correspond to ARNs or secret names in Secrets Manager storing PEM formatted keys. For example: ```toml -[[resolvers]] -regex = '.*' -substitution_string = '$0' +[[locations]] +regex = ".*" +substitution_string = "$0" -[resolvers.storage.keys] -location = 'SecretsManager' -private_key = 'private_key_secret_name' # pragma: allowlist secret -recipient_public_key = 'public_key_secret_name' +location.keys.key_location = "SecretsManager" +location.keys.private_key = "private_key_secret_name" # pragma: allowlist secret +location.keys.recipient_public_key = "public_key_secret_name" ``` The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure. Any of the storage types are supported, i.e. `Local`, `S3`, or `Url`. +### Log formatting + +The `RUST_LOG` variable is read to configure the level that trace logs are emitted. + +For example, the following indicates trace level for all htsget crates, and info level for all other crates: + +```sh +export RUST_LOG='info,htsget_lambda=trace,htsget_lambda=trace,htsget_config=trace,htsget_http=trace,htsget_search=trace,htsget_test=trace' +``` + +See [here][rust-log] for more information on setting this variable. + +The style of formatting can be configured by setting the following option: + +| Option | Description | Type | Default | +|---------------------------------------------------------|--------------------------------------|--------------------------------------------------------|----------| +| `formatting_style` | The style of log formatting to use. | One of `'Full'`, `'Compact'`, `'Pretty'`, or `'Json'` | `'Full'` | + +See [here][formatting-style] for more information on how these values look. + +### Environment variables + +Advanced configuration options also support environment variables. Generally, options separated by `.` in a config file +are separated by `_` in the corresponding environment variable. For example, to set the ticket server allow origins, +use `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS`. It is not recommended to set regex-based locations using environment +variables because the variables needs to contain the nested array structure of regex locations. + ### As a library This crate reads config files and environment variables using [figment], and accepts command-line arguments using clap. The main function for this is `from_config`, @@ -646,7 +424,7 @@ regex, and changing it by using a substitution string. [figment]: https://github.com/SergioBenitez/Figment -#### Feature flags +### Feature flags This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. @@ -657,6 +435,19 @@ This crate has the following features: This project is licensed under the [MIT license][license]. +[tracing]: https://github.com/tokio-rs/tracing +[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html +[formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters +[examples-config-files]: examples/config-files +[rustls]: https://github.com/rustls/rustls +[mkcert]: https://github.com/FiloSottile/mkcert +[htsget-actix]: ../htsget-actix +[htsget-axum]: ../htsget-axum +[htsget-lambda]: ../htsget-lambda +[tracing]: https://github.com/tokio-rs/tracing +[rust-log]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html +[formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters +[service-info]: https://samtools.github.io/hts-specs/htsget.html#ga4gh-service-info [path-addressing]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access [env-variables]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html [virtual-addressing]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index 0767a2c9..6eb5112e 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -14,7 +14,7 @@ pub struct RegexLocation { #[serde(with = "serde_regex")] regex: Regex, substitution_string: String, - location: Backend, + backend: Backend, guard: Option, } @@ -23,13 +23,13 @@ impl RegexLocation { pub fn new( regex: Regex, substitution_string: String, - location: Backend, + backend: Backend, guard: Option, ) -> Self { Self { regex, substitution_string, - location, + backend, guard, } } @@ -46,7 +46,7 @@ impl RegexLocation { /// Get the storage backend. pub fn backend(&self) -> &Backend { - &self.location + &self.backend } /// Get the allow guard. diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 9b12f524..c88cd36a 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -218,6 +218,7 @@ pub(crate) mod tests { use crate::types::Scheme; use figment::Jail; use http::uri::Authority; + use http::Uri; use serde::de::DeserializeOwned; use serde_json::json; diff --git a/htsget-config/src/config/service_info.rs b/htsget-config/src/config/service_info.rs index 33617cb6..2cd7bb3a 100644 --- a/htsget-config/src/config/service_info.rs +++ b/htsget-config/src/config/service_info.rs @@ -63,7 +63,7 @@ mod tests { fn service_info() { test_serialize_and_deserialize( r#" - id = { "id" = "1" } + id = { id = "1" } "#, ServiceInfo::new(HashMap::from_iter(vec![( "id".to_string(), diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index 34146238..3172de90 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -11,17 +11,14 @@ use std::path::PathBuf; #[derive(Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct C4GHLocal { - private_key: PathBuf, - recipient_public_key: PathBuf, + private: PathBuf, + public: PathBuf, } impl C4GHLocal { /// Create a new local C4GH key storage. - pub fn new(private_key: PathBuf, recipient_public_key: PathBuf) -> Self { - Self { - private_key, - recipient_public_key, - } + pub fn new(private: PathBuf, public: PathBuf) -> Self { + Self { private, public } } } @@ -29,8 +26,8 @@ impl TryFrom for C4GHKeys { type Error = Error; fn try_from(local: C4GHLocal) -> Result { - let private_key = get_private_key(local.private_key, Ok("".to_string()))?; - let recipient_public_key = get_public_key(local.recipient_public_key)?; + let private_key = get_private_key(local.private, Ok("".to_string()))?; + let recipient_public_key = get_public_key(local.public)?; let handle = tokio::spawn(async move { Ok(C4GHKeys::from_key_pair(private_key, recipient_public_key)) }); diff --git a/htsget-config/src/storage/c4gh/mod.rs b/htsget-config/src/storage/c4gh/mod.rs index 972ad6ec..260ef4f5 100644 --- a/htsget-config/src/storage/c4gh/mod.rs +++ b/htsget-config/src/storage/c4gh/mod.rs @@ -73,7 +73,7 @@ impl TryFrom for C4GHKeys { /// The location of C4GH keys. #[derive(Deserialize, Debug, Clone)] -#[serde(tag = "key_location", deny_unknown_fields)] +#[serde(tag = "kind", deny_unknown_fields)] #[non_exhaustive] pub enum C4GHKeyLocation { #[serde(alias = "file", alias = "FILE")] diff --git a/htsget-config/src/storage/c4gh/secrets_manager.rs b/htsget-config/src/storage/c4gh/secrets_manager.rs index fba485f5..fcba0900 100644 --- a/htsget-config/src/storage/c4gh/secrets_manager.rs +++ b/htsget-config/src/storage/c4gh/secrets_manager.rs @@ -18,18 +18,18 @@ use tempfile::TempDir; #[derive(Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct C4GHSecretsManager { - private_key: String, - recipient_public_key: String, + private: String, + public: String, #[serde(skip)] client: Option, } impl C4GHSecretsManager { /// Create a new C4GH secrets manager key storage. - pub fn new(private_key: String, recipient_public_key: String) -> Self { + pub fn new(private: String, public: String) -> Self { Self { - private_key, - recipient_public_key, + private, + public, client: None, } } @@ -69,10 +69,10 @@ impl C4GHSecretsManager { // Should not have to do this, but the Crypt4GH library expects a path. let tmp = TempDir::new()?; let private_key = tmp.path().join("private_key"); - Self::write_to_file(&private_key, self.private_key, &client).await?; + Self::write_to_file(&private_key, self.private, &client).await?; let recipient_public_key = tmp.path().join("public_key"); - Self::write_to_file(&recipient_public_key, self.recipient_public_key, &client).await?; + Self::write_to_file(&recipient_public_key, self.public, &client).await?; let private_key = get_private_key(private_key, Ok("".to_string()))?; let recipient_public_key = get_public_key(recipient_public_key)?; diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index cbf984ba..999ddec6 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -34,7 +34,7 @@ impl ResolvedId { /// Specify the storage backend to use as config values. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "backend", deny_unknown_fields)] +#[serde(tag = "kind", deny_unknown_fields)] #[non_exhaustive] pub enum Backend { #[serde(alias = "file", alias = "FILE")] @@ -129,8 +129,7 @@ pub(crate) mod tests { r#" [[locations]] regex = "regex" - [locations.location] - backend = "S3" + backend.kind = "S3" "#, |config| { assert!(matches!( From 3939bd34f3be7d09bba1e3e27aaaf96c5d93df8b Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 17 Dec 2024 14:29:57 +1100 Subject: [PATCH 10/12] refactor: fix tests and tidy docs --- Cargo.lock | 258 ++++++++++++------ README.md | 1 + htsget-actix/benches/request_benchmarks.rs | 2 +- htsget-axum/src/server/data.rs | 6 +- htsget-config/README.md | 74 ++--- .../examples/config-files/basic.toml | 5 - htsget-config/examples/config-files/c4gh.toml | 24 +- .../examples/config-files/default.toml | 71 +++-- .../examples/config-files/s3_storage.toml | 23 +- .../config-files/tls_data_server.toml | 17 +- .../config-files/tls_ticket_server.toml | 18 +- .../examples/config-files/url_storage.toml | 27 +- .../src/config/advanced/allow_guard.rs | 36 +++ htsget-config/src/config/advanced/cors.rs | 36 +++ .../src/config/advanced/regex_location.rs | 96 +++++-- htsget-config/src/config/mod.rs | 57 ++-- htsget-config/src/config/service_info.rs | 17 +- htsget-config/src/resolver.rs | 14 +- htsget-config/src/storage/c4gh/local.rs | 16 +- htsget-config/src/storage/mod.rs | 7 +- htsget-http/src/lib.rs | 4 +- htsget-search/src/bam_search.rs | 2 +- htsget-search/src/bcf_search.rs | 2 +- htsget-search/src/cram_search.rs | 2 +- htsget-search/src/vcf_search.rs | 2 +- htsget-storage/src/c4gh/storage.rs | 2 +- htsget-storage/src/lib.rs | 4 +- htsget-storage/src/local.rs | 6 +- htsget-test/Cargo.toml | 6 +- htsget-test/src/aws_mocks.rs | 4 +- htsget-test/src/http/server.rs | 2 +- 31 files changed, 545 insertions(+), 296 deletions(-) delete mode 100644 htsget-config/examples/config-files/basic.toml diff --git a/Cargo.lock b/Cargo.lock index 9e90172d..84f55faf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,7 +66,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand", - "sha1", + "sha1 0.10.6", "smallvec", "tokio", "tokio-util", @@ -242,7 +242,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "generic-array", ] @@ -579,14 +579,14 @@ dependencies = [ "bytes", "fastrand", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http-body 0.4.6", "lru", "once_cell", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.10.8", "tracing", "url", ] @@ -696,14 +696,14 @@ dependencies = [ "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.2.0", "once_cell", "p256", "percent-encoding", "ring", - "sha2", + "sha2 0.10.8", "subtle", "time", "tracing", @@ -712,9 +712,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "8aa8ff1492fd9fb99ae28e8467af0dbbb7c31512b16fabf1a0f10d7bb6ef78bb" dependencies = [ "futures-util", "pin-project-lite", @@ -737,8 +737,8 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.8", "tracing", ] @@ -833,9 +833,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "431a10d0e07e09091284ef04453dae4069283aa108d209974d67e77ae1caa658" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -849,7 +849,7 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "indexmap 2.7.0", "once_cell", @@ -882,9 +882,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "8ecbf4d5dfb169812e2b240a4350f15ad3c6b03a54074e5712818801615f2dc5" dependencies = [ "base64-simd", "bytes", @@ -968,7 +968,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "itoa", "matchit", @@ -1092,7 +1092,7 @@ checksum = "6aeac2e1fe888769f34f05ac343bbef98b14d1ffb292ab69d4608b3abc86f2a2" dependencies = [ "blowfish", "pbkdf2", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -1145,7 +1145,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1157,6 +1157,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.11.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd016a0ddc7cb13661bf5576073ce07330a693f8608a1320b4e20561cc12cdc" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1319,9 +1328,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" dependencies = [ "jobserver", "libc", @@ -1419,7 +1428,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "inout", "zeroize", ] @@ -1496,6 +1505,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ff6be19477a1bd5441f382916a89bc2a0b2c35db6d41e0f6e8538bf6d6463f" + [[package]] name = "convert_case" version = "0.4.0" @@ -1596,18 +1611,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1624,9 +1639,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1696,6 +1711,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b8ce8218c97789f16356e7896b3714f26c2ee1079b79c0b7ae7064bb9089fa" +dependencies = [ + "getrandom", + "hybrid-array", + "rand_core", +] + [[package]] name = "crypto_kx" version = "0.2.1" @@ -1795,7 +1821,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -1834,8 +1860,20 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.0-pre.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2e3d6615d99707295a9673e889bf363a04b2a466bd320c65a72536f7577379" +dependencies = [ + "block-buffer 0.11.0-rc.3", + "const-oid 0.10.0-rc.3", + "crypto-common 0.2.0-rc.1", "subtle", ] @@ -1889,7 +1927,7 @@ dependencies = [ "base16ct", "crypto-bigint 0.4.9", "der", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -2259,16 +2297,25 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4b1fb14e4df79f9406b434b60acef9f45c26c50062cccf1346c6103b8c47d58" +dependencies = [ + "digest 0.11.0-pre.9", ] [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2313,7 +2360,7 @@ dependencies = [ "htsget-search", "htsget-test", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "reqwest", "rustls 0.23.20", @@ -2569,11 +2616,20 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hybrid-array" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d35805454dc9f8662a98d6d61886ffe26bd465f5960e0e55345c70d5c0d2a9" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -2595,9 +2651,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -2622,7 +2678,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "rustls 0.21.12", "rustls-native-certs", @@ -2638,7 +2694,7 @@ checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "rustls 0.23.20", "rustls-pki-types", @@ -2659,7 +2715,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -2969,7 +3025,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "lambda_runtime", "mime", "percent-encoding", @@ -2995,7 +3051,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "http-serde", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "lambda_runtime_api_client", "pin-project", @@ -3021,7 +3077,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "tokio", "tower 0.4.13", @@ -3215,7 +3271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -3632,7 +3688,7 @@ checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" dependencies = [ "ecdsa", "elliptic-curve", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -3699,8 +3755,8 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", - "hmac", + "digest 0.10.7", + "hmac 0.12.1", ] [[package]] @@ -3933,9 +3989,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "memchr", "serde", @@ -3954,7 +4010,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.20", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.7", "tokio", "tracing", ] @@ -3973,7 +4029,7 @@ dependencies = [ "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.7", "tinyvec", "tracing", "web-time", @@ -4138,7 +4194,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-rustls 0.27.3", "hyper-util", "ipnet", @@ -4176,7 +4232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ "crypto-bigint 0.4.9", - "hmac", + "hmac 0.12.1", "zeroize", ] @@ -4325,9 +4381,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -4368,9 +4424,8 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "s3s" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa54e3b4b4791c8c62291516997866b4f265c3fcbfdbcdd0b8da62896fba8bfa" +version = "0.11.0-dev" +source = "git+https://github.com/Nugine/s3s#787319c5a6c6df905dc8d1b4a140635fef75d42d" dependencies = [ "arrayvec", "async-trait", @@ -4381,14 +4436,14 @@ dependencies = [ "chrono", "crc32c", "crc32fast", - "digest", + "digest 0.11.0-pre.9", "futures", "hex-simd", - "hmac", + "hmac 0.13.0-pre.4", "http-body 1.0.1", "http-body-util", "httparse", - "hyper 1.5.1", + "hyper 1.5.2", "itoa", "memchr", "mime", @@ -4399,11 +4454,12 @@ dependencies = [ "quick-xml", "serde", "serde_urlencoded", - "sha1", - "sha2", + "sha1 0.11.0-pre.4", + "sha2 0.11.0-pre.4", "smallvec", + "std-next", "sync_wrapper", - "thiserror 1.0.69", + "thiserror 2.0.7", "time", "tokio", "tracing", @@ -4414,16 +4470,15 @@ dependencies = [ [[package]] name = "s3s-aws" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13db1822175997bba0d1e398a772085b75791aa14e793819bec728ed8b394ff2" +version = "0.11.0-dev" +source = "git+https://github.com/Nugine/s3s#787319c5a6c6df905dc8d1b4a140635fef75d42d" dependencies = [ "async-trait", "aws-sdk-s3", "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-types-convert", - "hyper 1.5.1", + "hyper 1.5.2", "s3s", "sync_wrapper", "tracing", @@ -4432,9 +4487,8 @@ dependencies = [ [[package]] name = "s3s-fs" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b045f5ab67e8536d147ae48e038f6f533717fe5ddb0a68d0ba8f922ce7fb293f" +version = "0.11.0-dev" +source = "git+https://github.com/Nugine/s3s#787319c5a6c6df905dc8d1b4a140635fef75d42d" dependencies = [ "async-trait", "base64-simd", @@ -4445,12 +4499,12 @@ dependencies = [ "hex-simd", "md-5", "mime", - "nugine-rust-utils", "numeric_cast", "path-absolutize", "s3s", "serde_json", - "thiserror 1.0.69", + "std-next", + "thiserror 2.0.7", "time", "tokio", "tokio-util", @@ -4502,7 +4556,7 @@ dependencies = [ "password-hash", "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -4676,7 +4730,18 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9540978cef7a8498211c1b1c14e5ce920fe5bd524ea84f4a3d72d4602515ae93" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.0-pre.9", ] [[package]] @@ -4687,7 +4752,18 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "540c0893cce56cdbcfebcec191ec8e0f470dd1889b6e7a0b503e310a94a168f5" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.0-pre.9", ] [[package]] @@ -4720,7 +4796,7 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest", + "digest 0.10.7", "rand_core", ] @@ -4783,6 +4859,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "std-next" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87854dde78837ff867561b22d873c53c25938d4b152afba2fa6fc48cb406c099" +dependencies = [ + "simdutf8", + "thiserror 2.0.7", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4870,11 +4956,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.7", ] [[package]] @@ -4890,9 +4976,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" dependencies = [ "proc-macro2", "quote", @@ -5307,7 +5393,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "subtle", ] diff --git a/README.md b/README.md index ba3bc122..5e1e990f 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Other directories contain further applications or data: - [deploy]: Deployments for htsget-rs. [axum]: https://github.com/tokio-rs/axum +[htsget-axum]: htsget-axum [htsget-config]: htsget-config [htsget-actix]: htsget-actix [htsget-http]: htsget-http diff --git a/htsget-actix/benches/request_benchmarks.rs b/htsget-actix/benches/request_benchmarks.rs index 4b60d5e9..60af6b82 100644 --- a/htsget-actix/benches/request_benchmarks.rs +++ b/htsget-actix/benches/request_benchmarks.rs @@ -150,7 +150,7 @@ fn start_htsget_rs() -> (DropGuard, String) { let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let data_server = config.data_server().clone().as_data_server_config(); + let data_server = config.data_server().as_data_server_config().unwrap(); let htsget_rs_ticket_url = format!("http://{}", data_server.addr()); query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); diff --git a/htsget-axum/src/server/data.rs b/htsget-axum/src/server/data.rs index d74d5b9e..fd4240c4 100644 --- a/htsget-axum/src/server/data.rs +++ b/htsget-axum/src/server/data.rs @@ -238,7 +238,7 @@ mod tests { test_cors_simple_request_uri( &DataTestServer::default(), - &format!("http://localhost:{port}/data/key1"), + &format!("http://localhost:{port}/key1"), ) .await; } @@ -251,7 +251,7 @@ mod tests { test_cors_preflight_request_uri( &DataTestServer::default(), - &format!("http://localhost:{port}/data/key1"), + &format!("http://localhost:{port}/key1"), ) .await; } @@ -298,7 +298,7 @@ mod tests { let request = test_server .request() .method(Method::GET) - .uri(format!("{scheme}://localhost:{port}/data/key1")); + .uri(format!("{scheme}://localhost:{port}/key1")); let response = test_server.test_server(request, "".to_string()).await; assert!(response.is_success()); diff --git a/htsget-config/README.md b/htsget-config/README.md index 1dd5c0dc..27555efb 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -24,7 +24,7 @@ locations = "file://data" Then launch the server using the config file: ```sh -cargo run -p htsget-axum -- --config +cargo run --all-features -p htsget-axum -- --config ``` This will serve files under the [`data`][data] directory: @@ -40,7 +40,7 @@ files on s3, which returns pre-signed URLs for tickets: locations = "s3://bucket" ``` -or on a remote HTTP server: +or on a remote HTTP server (either `http://` or `https://`): ```toml locations = "https://example.com" @@ -76,6 +76,16 @@ The data server process can be disabled by setting it to `None` if no file locat data_server = "None" ``` +> [!NOTE] +> For S3 locations, the bucket is not included in the request to htsget-rs. To include the bucket as well, +> see deriving the bucket from the first capture group in [advanced config](#bucket). + +> [!IMPORTANT] +> Some parts of htsget-rs require extra feature flags for conditional compilation, that's why the examples specify +> using `--all-features`. Notably, `--features s3-storage` enables the `S3` location type, and `--features url-storage` +> enabled the remote HTTP server location type. If using a subset of features, for example S3 locations only, then +> a single feature can be enabled instead of using `--all-features`. + ### Server config htsget-rs spawn up to two server instances - the ticket server responds to the initial htsget request, and optionally, @@ -133,7 +143,7 @@ The following section describes advanced configuration which is more flexible, b ### Regex-based location Instead of the simple path-based locations described above, htsget-rs supports arbitrary regex-based id resolution. -This allows matching an [`id`][id] using, which is everything after `reads/` or `variants/` in the http path, and mapping +This allows matching an [`id`][id], which is everything after `reads/` or `variants/` in the http path, and mapping it to a location using regex substitution. To create a regex location, add a `[[locations]]` array of tables, and set the following options: @@ -147,27 +157,29 @@ For example, below is a `regex` option which matches a `/` between two groups, a in between the groups with the `substitution_string`: ```toml -[[resolvers]] +[[locations]] regex = '(?P.*?)/(?P.*)' substitution_string = '$group1/data/$group2' ``` -This would mean that a request to `http://localhost:8080/reads/some_id/file` would search for files at `some_id/data/file.bam` and `some_id/data/file.bam.bai`. +This would mean that a request to `http://localhost:8080/reads/some_id/file` would search for files at `some_id/data/file.bam`. The regex locations also have access to further configuration of storage locations for `file://`, `s3://`, or `http://` locations. These are called `File`, `S3`, and `Url` respectively. To manually configure `File` locations, set `backend.kind = "File"`, and specify any additional options from below the `backend` table: -| Option | Description | Type | Default | -|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| -| `scheme` | The scheme present on URL tickets. | Either `'Http'` or `'Https'` | `'Http'` | -| `authority` | The authority present on URL tickets. This should likely match the `data_server.addr`. | URL authority | `'127.0.0.1:8081'` | -| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server.local_path`. | Filesystem path | `'./'` | +| Option | Description | Type | Default | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| +| `scheme` | The scheme present on URL tickets. | Either `'Http'` or `'Https'` | `'Http'` | +| `authority` | The authority present on URL tickets. This should likely match the `data_server.addr`. | URL authority | `'127.0.0.1:8081'` | +| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server.local_path`. | Filesystem path | `'./'` | For example: ```toml +data_server.addr = "127.0.0.1:8000" + [[locations]] regex = ".*" substitution_string = "$0" @@ -180,11 +192,11 @@ backend.local_path = "path" To manually configure `S3` locations, set `backend.kind = "S3"`, and specify options from below under the `backend` table: -| Option | Description | Type | Default | -|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------| -| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `location` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. | -| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. | -| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | +| Option | Description | Type | Default | +|------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------| +| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `location` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. | +| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. | +| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | For example, the following backend manually sets the `bucket` and uses path style requests: @@ -242,6 +254,8 @@ forward_headers = false If there is an overlap in regex matches, the first location specified will be the one used. +Additional config file examples are available under [`example/config-files`][examples-config-files]. + ### Allow guard Additionally, locations support resolving IDs based on the other fields present in a query. @@ -256,8 +270,8 @@ This component can be configured by setting the `guard` table with: | `allow_tags` | Resolve the query ID if the query also contains the tags set by this option. | Array of tags or `'All'` | `'All'` | | `allow_formats` | Resolve the query ID if the query is one of the formats specified by this option. | An array of formats containing `'BAM'`, `'CRAM'`, `'VCF'`, or `'BCF'` | `['BAM', 'CRAM', 'VCF', 'BCF']` | | `allow_classes` | Resolve the query ID if the query is one of the classes specified by this option. | An array of classes containing eithr `'body'` or `'header'` | `['body', 'header']` | -| `allow_interval_start` | Resolve the query ID if the query reference start position is at least this option. | Unsigned 32-bit integer start position, 0-based, inclusive | Not set, allows all start positions | -| `allow_interval_end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive | Not set, allows all end positions | +| `allow_interval.start` | Resolve the query ID if the query reference start position is at least this option. | Unsigned 32-bit integer start position, 0-based, inclusive | Not set, allows all start positions | +| `allow_interval.end` | Resolve the query ID if the query reference end position is at most this option. | Unsigned 32-bit integer end position, 0-based exclusive | Not set, allows all end positions | For example, match only if the request queries `chr1` with positions between `100` and `1000`: @@ -270,8 +284,8 @@ backend.kind = "S3" backend.bucket = "bucket" guard.allow_reference_names = ["chr1"] -guard.allow_interval_start = 100 -guard.allow_interval_end = 1000 +guard.allow_interval.start = 100 +guard.allow_interval.end = 1000 ``` ### Server configuration @@ -313,7 +327,7 @@ CORS can also be configured for the data and ticket servers by specifying the `c ticket_server.cors.allow_credentials = false ticket_server.cors.allow_origins = "Mirror" ticket_server.cors.allow_headers = "All" -ticket_server.cors.allow_methods = ['GET', 'POST'] +ticket_server.cors.allow_methods = ["GET", "POST"] ticket_server.cors.max_age = 86400 ticket_server.cors.expose_headers = [] ``` @@ -381,9 +395,9 @@ For example: regex = ".*" substitution_string = "$0" -location.keys.key_location = "SecretsManager" -location.keys.private_key = "private_key_secret_name" # pragma: allowlist secret -location.keys.recipient_public_key = "public_key_secret_name" +location.keys.kind = "SecretsManager" +location.keys.private = "private_key_secret_name" # pragma: allowlist secret +location.keys.public = "public_key_secret_name" ``` The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure. @@ -414,21 +428,22 @@ See [here][formatting-style] for more information on how these values look. Advanced configuration options also support environment variables. Generally, options separated by `.` in a config file are separated by `_` in the corresponding environment variable. For example, to set the ticket server allow origins, use `HTSGET_TICKET_SERVER_CORS_ALLOW_ORIGINS`. It is not recommended to set regex-based locations using environment -variables because the variables needs to contain the nested array structure of regex locations. +variables because the variables needs to contain the nested array structure of storage backends. ### As a library This crate reads config files and environment variables using [figment], and accepts command-line arguments using clap. The main function for this is `from_config`, -which is used to obtain the `Config` struct. The crate also contains the `regex_resolver` abstraction, which is used for matching a query ID with -regex, and changing it by using a substitution string. +which is used to obtain the `Config` struct. The crate also contains the `resolver` abstraction, which is used for matching a query ID with +regex, and changing it by using a substitution string. Advanced configuration options are specified in the [`advanced.rs`][advanced] submodule. +[advanced]: src/config/advanced/mod.rs [figment]: https://github.com/SergioBenitez/Figment ### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3-storage`: used to enable `S3` functionality. +* `url-storage`: used to enable `Url` functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License @@ -440,7 +455,6 @@ This project is licensed under the [MIT license][license]. [formatting-style]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/index.html#formatters [examples-config-files]: examples/config-files [rustls]: https://github.com/rustls/rustls -[mkcert]: https://github.com/FiloSottile/mkcert [htsget-actix]: ../htsget-actix [htsget-axum]: ../htsget-axum [htsget-lambda]: ../htsget-lambda @@ -458,7 +472,5 @@ This project is licensed under the [MIT license][license]. [data-c4gh]: ../data/c4gh [secrets-manager]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html [id]: https://samtools.github.io/hts-specs/htsget.html#url-parameters -[basic]: examples/config-files/basic.toml -[data-server]: README.md#data-server-config [toml]: https://toml.io/en/ [data]: ../data \ No newline at end of file diff --git a/htsget-config/examples/config-files/basic.toml b/htsget-config/examples/config-files/basic.toml deleted file mode 100644 index 44d4f351..00000000 --- a/htsget-config/examples/config-files/basic.toml +++ /dev/null @@ -1,5 +0,0 @@ -# An example of running htsget-rs. -# Run with `cargo run --all-features -- --config htsget-config/examples/config-files/basic.toml` -locations = ["file://data/bam", "file://data/cram"] - -# Uses the first capture group in the regex as the bucket. diff --git a/htsget-config/examples/config-files/c4gh.toml b/htsget-config/examples/config-files/c4gh.toml index 4a350fec..00ccd1e1 100644 --- a/htsget-config/examples/config-files/c4gh.toml +++ b/htsget-config/examples/config-files/c4gh.toml @@ -1,23 +1,23 @@ # An example of running htsget-rs with Crypt4GH enabled. # Run with `cargo run -p htsget-axum --features experimental -- --config htsget-config/examples/config-files/c4gh.toml` -ticket_server_addr = "127.0.0.1:8080" -data_server_addr = "127.0.0.1:8081" +ticket_server.addr = "127.0.0.1:8080" +data_server.addr = "127.0.0.1:8081" [[resolvers]] regex = ".*" substitution_string = "$0" -[resolvers.storage] -backend = 'Local' +[locations.backend] +kind = "File" -[resolvers.storage.keys] -location = "Local" -private_key = "data/c4gh/keys/bob.sec" # pragma: allowlist secret -recipient_public_key = "data/c4gh/keys/alice.pub" +[locations.backend.keys] +kind = "Local" +private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret +public = "data/c4gh/keys/alice.pub" # Or, use AWS secrets manager to store keys. -#[resolvers.storage.keys] -#location = "SecretsManager" -#private_key = "htsget/test_c4gh_private_key" # pragma: allowlist secret -#recipient_public_key = "htsget/test_c4gh_public_key" +#[locations.backend.keys] +#kind = "SecretsManager" +#private = "htsget/test_c4gh_private_key" # pragma: allowlist secret +#public = "htsget/test_c4gh_public_key" diff --git a/htsget-config/examples/config-files/default.toml b/htsget-config/examples/config-files/default.toml index 93ea3608..6845e790 100644 --- a/htsget-config/examples/config-files/default.toml +++ b/htsget-config/examples/config-files/default.toml @@ -1,40 +1,37 @@ # Config generated by running `cargo run -p htsget-axum -- -p` formatting_style = "Full" -ticket_server_addr = "127.0.0.1:8080" -ticket_server_cors_allow_credentials = false -ticket_server_cors_allow_origins = ["http://localhost:8080"] -ticket_server_cors_allow_headers = "All" -ticket_server_cors_allow_methods = "All" -ticket_server_cors_max_age = 86400 -ticket_server_cors_expose_headers = [] -data_server_enabled = true -data_server_addr = "127.0.0.1:8081" -data_server_local_path = "./" -data_server_serve_at = "" -data_server_cors_allow_credentials = false -data_server_cors_allow_origins = ["http://localhost:8080"] -data_server_cors_allow_headers = "All" -data_server_cors_allow_methods = "All" -data_server_cors_max_age = 86400 -data_server_cors_expose_headers = [] - -[[resolvers]] -regex = ".*" -substitution_string = "$0" -storage = "Local" - -[resolvers.allow_guard] -allow_reference_names = "All" -allow_fields = "All" -allow_tags = "All" -allow_formats = [ - "BAM", - "CRAM", - "VCF", - "BCF", -] -allow_classes = [ - "body", - "header", -] + +[ticket_server] +addr = "127.0.0.1:8080" + +[ticket_server.cors] +allow_credentials = false +allow_origins = "Mirror" +allow_headers = "Mirror" +allow_methods = "Mirror" +max_age = 2592000 +expose_headers = "All" + +[data_server] +addr = "127.0.0.1:8081" +local_path = "./" + +[data_server.cors] +allow_credentials = false +allow_origins = "Mirror" +allow_headers = "Mirror" +allow_methods = "Mirror" +max_age = 2592000 +expose_headers = "All" + +[service_info] + +[[locations]] +prefix = "" + +[locations.backend] +kind = "File" +scheme = "HTTP" +authority = "127.0.0.1:8081" +local_path = "./" diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 5cde4dff..7948e457 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -1,19 +1,18 @@ # An example for a server which uses s3 storage with data located in "bucket". # Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/s3_storage.toml` -ticket_server_cors_allow_headers = "All" -ticket_server_cors_allow_methods = "All" -ticket_server_cors_allow_credentials = true -ticket_server_cors_max_age = 300 +ticket_server.cors.allow_headers = "All" +ticket_server.cors.allow_methods = "All" +ticket_server.cors.allow_credentials = true +ticket_server.cors.max_age = 300 -data_server_enabled = false +data_server = "None" -[[resolvers]] -regex = '^(bucket)/(?P.*)$' -substitution_string = '$key' -storage.backend = 'S3' +[[locations]] +regex = "^(bucket)/(?P.*)$" +substitution_string = "$key" +backend.kind = "S3" # Or, set the bucket manually -#[resolvers.storage] -#backend = 'S3' -#bucket = 'bucket' +#backend.kind = "S3" +#backend.bucket = "bucket" diff --git a/htsget-config/examples/config-files/tls_data_server.toml b/htsget-config/examples/config-files/tls_data_server.toml index d2e4316e..5d8987d8 100644 --- a/htsget-config/examples/config-files/tls_data_server.toml +++ b/htsget-config/examples/config-files/tls_data_server.toml @@ -1,16 +1,13 @@ # An example config file for a TLS data server that uses a local storage backend. # Run with `cargo run -p htsget-axum -- --config htsget-config/examples/config-files/tls_data_server.toml` -ticket_server_addr = "0.0.0.0:8080" -data_server_addr = "0.0.0.0:8081" -data_server_cors_allow_origins = "All" -data_server_tls.cert = "cert.pem" -data_server_tls.key = "key.pem" +ticket_server.addr = "0.0.0.0:8080" +data_server.addr = "0.0.0.0:8081" +data_server.cors.allow_origins = "All" +data_server.tls.cert = "cert.pem" +data_server.tls.key = "key.pem" -[[resolvers]] +[[locations]] regex = ".*" substitution_string = "$0" - -[resolvers.storage] -backend = 'Local' -use_data_server_config = true +backend.kind = "Local" diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index 9bd196ff..d73bb596 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -1,16 +1,14 @@ # An example config file for a TLS ticket server that uses S3 as a storage backend. # Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/tls_ticket_server.toml` -ticket_server_addr = "0.0.0.0:8080" -ticket_server_cors_allow_origins = "All" -ticket_server_tls.cert = "cert.pem" -ticket_server_tls.key = "key.pem" -data_server_addr = "0.0.0.0:8081" +ticket_server.addr = "0.0.0.0:8080" +ticket_server.cors_allow_origins = "All" +ticket_server.tls.cert = "cert.pem" +ticket_server.tls.key = "key.pem" +data_server.addr = "0.0.0.0:8081" -[[resolvers]] +[[locations]] regex = ".*" substitution_string = "$0" - -[resolvers.storage] -backend = 'S3' -bucket = "bucket" +backend.kind = "S3" +backend.bucket = "bucket" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index 372b0080..078a41b4 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -3,27 +3,26 @@ # `cargo run -p htsget-axum --features url-storage -- --config htsget-config/examples/config-files/url_storage.toml` # in the project directory. -ticket_server_addr = "127.0.0.1:8082" -ticket_server_cors_allow_origins = "All" +ticket_server.addr = "127.0.0.1:8082" +ticket_server.cors.allow_origins = "All" -ticket_server_cert = "cert.pem" -ticket_server_key = "key.pem" +ticket_server.cert = "cert.pem" +ticket_server.key = "key.pem" -data_server_enabled = false +data_server = "None" -[[resolvers]] +[[locations]] regex = ".*" substitution_string = "$0" -[resolvers.storage] -backend = 'Url' -url = "http://127.0.0.1:8081" -response_url = "https://127.0.0.1:8081" -forward_headers = true +backend.kind = "Url" +backend.url = "http://127.0.0.1:8081" +backend.response_url = "https://127.0.0.1:8081" +backend.forward_headers = true # Set client authentication -#tls.key = "key.pem" -#tls.cert = "cert.pem" +#backend.tls.key = "key.pem" +#backend.tls.cert = "cert.pem" # Set root certificates -#tls.root_store = "cert.pem" +#backend.tls.root_store = "cert.pem" diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs index 17e6b14c..103b8675 100644 --- a/htsget-config/src/config/advanced/allow_guard.rs +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -145,6 +145,8 @@ impl QueryAllowed for AllowGuard { mod tests { use super::*; use crate::config::tests::test_serialize_and_deserialize; + #[cfg(feature = "s3-storage")] + use crate::config::Config; use crate::types::Class::Header; #[test] @@ -258,6 +260,40 @@ mod tests { ); } + #[cfg(feature = "s3-storage")] + #[test] + fn allow_guard() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = ".*" + substitution_string = "$0" + + backend.kind = "S3" + backend.bucket = "bucket" + + guard.allow_reference_names = ["chr1"] + guard.allow_interval.start = 100 + guard.allow_interval.end = 1000 + "#, + AllowGuard { + allow_reference_names: ReferenceNames::List(HashSet::from_iter(vec!["chr1".to_string()])), + allow_interval: Interval::new(Some(100), Some(1000)), + ..Default::default() + }, + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + + AllowGuard { + allow_reference_names: location.guard().unwrap().allow_reference_names.clone(), + allow_interval: location.guard().unwrap().allow_interval, + ..Default::default() + } + }, + ); + } + #[test] fn query_allowed() { let guard = AllowGuard { diff --git a/htsget-config/src/config/advanced/cors.rs b/htsget-config/src/config/advanced/cors.rs index 92e6c640..0e8618d7 100644 --- a/htsget-config/src/config/advanced/cors.rs +++ b/htsget-config/src/config/advanced/cors.rs @@ -233,6 +233,7 @@ impl Default for CorsConfig { mod tests { use super::*; use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; use http::Method; use toml::de::Error; @@ -284,6 +285,41 @@ mod tests { ); } + #[test] + fn cors_config() { + test_serialize_and_deserialize( + r#" + ticket_server.cors.allow_credentials = false + ticket_server.cors.allow_origins = "Mirror" + ticket_server.cors.allow_headers = "All" + data_server.cors.allow_methods = ["GET", "POST"] + data_server.cors.max_age = 86400 + data_server.cors.expose_headers = [] + "#, + ( + false, + AllowType::Tagged(TaggedAllowTypes::Mirror), + AllowType::Tagged(TaggedAllowTypes::All), + AllowType::List(vec!["GET".parse().unwrap(), "POST".parse().unwrap()]), + 86400, + AllowType::List(vec![]), + ), + |result: Config| { + let ticket_cors = result.ticket_server().cors(); + let data_cors = result.data_server().as_data_server_config().unwrap().cors(); + + ( + ticket_cors.allow_credentials, + ticket_cors.allow_origins.clone(), + ticket_cors.allow_headers.clone(), + data_cors.allow_methods.clone(), + data_cors.max_age, + data_cors.expose_headers.clone(), + ) + }, + ); + } + #[test] fn tagged_any_allow_type_err_on_mirror() { let allow_type_method = "expose_headers = \"Mirror\""; diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index 6eb5112e..d8bff741 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -74,22 +74,25 @@ impl From for LocationEither { #[cfg(test)] mod tests { - use super::*; use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; #[test] fn regex_location_file() { test_serialize_and_deserialize( r#" + [[locations]] regex = "123-.*" substitution_string = "123" "#, ("123-.*".to_string(), "123".to_string()), - |result: RegexLocation| { - result.backend().as_file().unwrap(); + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + location.backend().as_file().unwrap(); ( - result.regex().as_str().to_string(), - result.substitution_string().to_string(), + location.regex().as_str().to_string(), + location.substitution_string().to_string(), ) }, ); @@ -100,16 +103,25 @@ mod tests { fn regex_location_s3() { test_serialize_and_deserialize( r#" + [[locations]] regex = "123-.*" substitution_string = "123" - location.backend = "S3" + backend.kind = "S3" + backend.bucket = "bucket" "#, - ("123-.*".to_string(), "123".to_string()), - |result: RegexLocation| { - result.backend().as_s3().unwrap(); + ( + "123-.*".to_string(), + "123".to_string(), + "bucket".to_string(), + ), + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + let backend = location.backend().as_s3().unwrap(); ( - result.regex().as_str().to_string(), - result.substitution_string().to_string(), + location.regex().as_str().to_string(), + location.substitution_string().to_string(), + backend.bucket().to_string(), ) }, ); @@ -120,24 +132,72 @@ mod tests { fn regex_location_url() { test_serialize_and_deserialize( r#" + [[locations]] regex = "123-.*" substitution_string = "123" - [location] - backend = "Url" - url = "https://example.com" + backend.kind = "Url" + backend.url = "https://example.com" "#, ( "123-.*".to_string(), "123".to_string(), "https://example.com/".to_string(), ), - |result: RegexLocation| { - let url = result.backend().as_url().unwrap(); + |result: Config| { + let location = result.locations.into_inner(); + let location = location[0].as_regex().unwrap(); + let url = location.backend().as_url().unwrap(); + + ( + location.regex().as_str().to_string(), + location.substitution_string().to_string(), + url.url().to_string(), + ) + }, + ); + } + + #[cfg(all(feature = "url-storage", feature = "s3-storage"))] + #[test] + fn regex_location_multiple() { + test_serialize_and_deserialize( + r#" + [[locations]] + regex = "prefix/(?P.*)$" + substitution_string = "$key" + backend.kind = "S3" + backend.bucket = "bucket" + backend.path_style = true + + [[locations]] + regex = ".*" + substitution_string = "$0" + backend.kind = "Url" + backend.url = "http://localhost:8080" + backend.forward_headers = false + "#, + ( + "prefix/(?P.*)$".to_string(), + "$key".to_string(), + "bucket".to_string(), + ".*".to_string(), + "$0".to_string(), + "http://localhost:8080/".to_string(), + ), + |result: Config| { + let location = result.locations.into_inner(); + let location_one = location[0].as_regex().unwrap(); + let s3 = location_one.backend().as_s3().unwrap(); + let location_two = location[1].as_regex().unwrap(); + let url = location_two.backend().as_url().unwrap(); ( - result.regex().as_str().to_string(), - result.substitution_string().to_string(), + location_one.regex().as_str().to_string(), + location_one.substitution_string().to_string(), + s3.bucket().to_string(), + location_two.regex().as_str().to_string(), + location_two.substitution_string().to_string(), url.url().to_string(), ) }, diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index c88cd36a..9e9d36aa 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -109,7 +109,7 @@ impl Config { let args = Args::from_arg_matches(&Args::augment_args(augment_args).get_matches()) .map_err(|err| ArgParseError(err.to_string()))?; - if !args.config.as_ref().is_some_and(|path| path.exists()) { + if args.config.as_ref().is_some_and(|path| !path.exists()) { return Err(ParseError("config file not found".to_string())); } @@ -218,6 +218,7 @@ pub(crate) mod tests { use crate::types::Scheme; use figment::Jail; use http::uri::Authority; + #[cfg(feature = "url-storage")] use http::Uri; use serde::de::DeserializeOwned; use serde_json::json; @@ -522,14 +523,29 @@ pub(crate) mod tests { data_server.local_path = "path" [[locations]] - backend = "File" + regex = "123" + backend.kind = "File" + backend.local_path = "path" "#, |config| { assert_eq!(config.locations().len(), 1); let config = config.locations.into_inner(); let regex = config[0].as_regex().unwrap(); assert!(matches!(regex.backend(), - Backend::File(file) if file.local_path() == "path" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); + Backend::File(file) if file.local_path() == "path" && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8081"))); + }, + ); + } + + #[test] + fn simple_locations_env() { + test_config_from_env( + vec![ + ("HTSGET_DATA_SERVER_ADDR", "127.0.0.1:8080"), + ("HTSGET_LOCATIONS", "[file://data/bam, file://data/cram]"), + ], + |config| { + assert_multiple(config); }, ); } @@ -593,19 +609,11 @@ pub(crate) mod tests { fn simple_locations_multiple() { test_config_from_file( r#" + data_server.addr = "127.0.0.1:8080" locations = ["file://data/bam", "file://data/cram"] "#, |config| { - assert_eq!(config.locations().len(), 2); - let config = config.locations.into_inner(); - - let location = config[0].as_simple().unwrap(); - assert_eq!(location.prefix(), "bam"); - assert_file_location(location, "data"); - - let location = config[1].as_simple().unwrap(); - assert_eq!(location.prefix(), "cram"); - assert_file_location(location, "data"); + assert_multiple(config); }, ); } @@ -615,11 +623,12 @@ pub(crate) mod tests { fn simple_locations_multiple_mixed() { test_config_from_file( r#" + data_server.addr = "127.0.0.1:8080" data_server.local_path = "root" locations = ["file://dir_one/bam", "file://dir_two/cram", "s3://bucket/vcf"] "#, |config| { - assert_eq!(config.locations().len(), 2); + assert_eq!(config.locations().len(), 3); let config = config.locations.into_inner(); let location = config[0].as_simple().unwrap(); @@ -645,14 +654,26 @@ pub(crate) mod tests { data_server = "None" "#, |config| { - assert!(matches!( - config.data_server().as_data_server_config(), - Err(_) - )); + assert!(config.data_server().as_data_server_config().is_err()); }, ); } + fn assert_multiple(config: Config) { + assert_eq!(config.locations().len(), 2); + let config = config.locations.into_inner(); + + println!("{:#?}", config); + + let location = config[0].as_simple().unwrap(); + assert_eq!(location.prefix(), "bam"); + assert_file_location(location, "data"); + + let location = config[1].as_simple().unwrap(); + assert_eq!(location.prefix(), "cram"); + assert_file_location(location, "data"); + } + fn assert_file_location(location: &Location, local_path: &str) { assert!(matches!(location.backend(), Backend::File(file) if file.local_path() == local_path && file.scheme() == Scheme::Http && file.authority() == &Authority::from_static("127.0.0.1:8080"))); diff --git a/htsget-config/src/config/service_info.rs b/htsget-config/src/config/service_info.rs index 2cd7bb3a..1f7d89d9 100644 --- a/htsget-config/src/config/service_info.rs +++ b/htsget-config/src/config/service_info.rs @@ -57,19 +57,24 @@ impl<'de> Deserialize<'de> for ServiceInfo { mod tests { use super::*; use crate::config::tests::test_serialize_and_deserialize; + use crate::config::Config; use serde_json::json; #[test] fn service_info() { test_serialize_and_deserialize( r#" - id = { id = "1" } + service_info.environment = "dev" + service_info.organization = { name = "name", url = "https://example.com/" } "#, - ServiceInfo::new(HashMap::from_iter(vec![( - "id".to_string(), - json!({"id": "1"}), - )])), - |result| result, + HashMap::from_iter(vec![ + ("environment".to_string(), json!("dev")), + ( + "organization".to_string(), + json!({ "name": "name", "url": "https://example.com/" }), + ), + ]), + |result: Config| result.service_info.0, ); } } diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index 28731bf9..1f084b7b 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -4,6 +4,7 @@ use crate::storage; use crate::storage::{Backend, ResolvedId}; use crate::types::{Query, Response, Result}; use async_trait::async_trait; +use std::path::PathBuf; use tracing::instrument; /// A trait which matches the query id, replacing the match in the substitution text. @@ -85,7 +86,12 @@ impl IdResolver for LocationEither { match self { LocationEither::Simple(location) => { if query.id().starts_with(location.prefix()) { - return Some(ResolvedId::new(query.id().to_string())); + return Some(ResolvedId::new( + PathBuf::from(location.prefix()) + .join(query.id()) + .to_str()? + .to_string(), + )); } } LocationEither::Regex(regex_location) => { @@ -377,7 +383,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-1", Bam)) .unwrap() .into_inner(), - "id-1" + "id-1/id-1" ); assert_eq!( resolver @@ -385,7 +391,7 @@ mod tests { .resolve_id(&Query::new_with_default_request("id-2", Bam)) .unwrap() .into_inner(), - "id-2" + "id-2/id-2" ); } @@ -444,7 +450,7 @@ mod tests { vec![( "HTSGET_LOCATIONS", "[{ regex=regex, substitution_string=substitution_string, \ - location={ backend=S3, bucket=bucket }, \ + backend={ kind=S3, bucket=bucket }, \ guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ allow_formats=[BAM], allow_classes=[body], allow_interval={ start=100, \ end=1000 } } }]", diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index 3172de90..696abc8b 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -71,13 +71,13 @@ mod tests { [[locations]] regex = "regex" - [locations.location] + [locations.backend] {} - [locations.location.keys] - key_location = "File" - private_key = "{}" - recipient_public_key = "{}" + [locations.backend.keys] + kind = "File" + private = "{}" + public = "{}" "#, storage_config, private_key.to_string_lossy(), @@ -90,7 +90,7 @@ mod tests { } #[tokio::test] async fn config_local_storage_c4gh() { - test_c4gh_storage_config(r#"backend = "File""#, |config| { + test_c4gh_storage_config(r#"kind = "File""#, |config| { assert!(matches!( config.locations().first().unwrap().backend(), Backend::File(file) if file.keys().is_some() @@ -103,7 +103,7 @@ mod tests { async fn config_s3_storage_c4gh() { test_c4gh_storage_config( r#" - backend = "S3" + kind = "S3" bucket = "bucket" "#, |config| { @@ -120,7 +120,7 @@ mod tests { async fn config_url_storage_c4gh() { test_c4gh_storage_config( r#" - backend = "Url" + kind = "Url" url = "https://example.com/" response_url = "https://example.com/" forward_headers = false diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index 999ddec6..74f10b7d 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -97,8 +97,7 @@ pub(crate) mod tests { r#" [[locations]] regex = "regex" - [locations.location] - backend = "File" + backend.kind = "File" "#, |config| { assert!(matches!( @@ -112,7 +111,7 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_local_env() { test_config_from_env( - vec![("HTSGET_LOCATIONS", "[{location={ backend=File }}]")], + vec![("HTSGET_LOCATIONS", "[{backend={ kind=File }}]")], |config| { assert!(matches!( config.locations().first().unwrap().backend(), @@ -144,7 +143,7 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_s3_env() { test_config_from_env( - vec![("HTSGET_LOCATIONS", "[{location={ backend=S3 }}]")], + vec![("HTSGET_LOCATIONS", "[{backend={ kind=S3 }}]")], |config| { assert!(matches!( config.locations().first().unwrap().backend(), diff --git a/htsget-http/src/lib.rs b/htsget-http/src/lib.rs index 9f0b242e..8130d1fb 100644 --- a/htsget-http/src/lib.rs +++ b/htsget-http/src/lib.rs @@ -240,7 +240,7 @@ mod tests { JsonResponse::from(Response::new( Vcf, vec![ - Url::new("http://127.0.0.1:8081/data/vcf/sample1-bcbio-cancer.vcf.gz".to_string()) + Url::new("http://127.0.0.1:8081/vcf/sample1-bcbio-cancer.vcf.gz".to_string()) .with_headers(headers), ], )) @@ -250,7 +250,7 @@ mod tests { JsonResponse::from(Response::new( Bam, vec![ - Url::new("http://127.0.0.1:8081/data/bam/htsnexus_test_NA12878.bam".to_string()) + Url::new("http://127.0.0.1:8081/bam/htsnexus_test_NA12878.bam".to_string()) .with_headers(headers), ], )) diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 1fc2ac1b..4d80587a 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -641,7 +641,7 @@ pub(crate) mod tests { } pub(crate) fn expected_url() -> String { - "http://127.0.0.1:8081/data/htsnexus_test_NA12878.bam".to_string() + "http://127.0.0.1:8081/htsnexus_test_NA12878.bam".to_string() } pub(crate) fn expected_eof_url() -> Url { diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index c52c373c..d4431839 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -463,6 +463,6 @@ mod tests { } fn expected_url(name: &str) -> String { - format!("http://127.0.0.1:8081/data/{name}.bcf") + format!("http://127.0.0.1:8081/{name}.bcf") } } diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index 07251f3f..86121085 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -648,7 +648,7 @@ mod tests { } fn expected_url() -> String { - "http://127.0.0.1:8081/data/htsnexus_test_NA12878.cram".to_string() + "http://127.0.0.1:8081/htsnexus_test_NA12878.cram".to_string() } pub(crate) fn expected_eof_url() -> Url { diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index 032e0c27..d83209e1 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -467,6 +467,6 @@ pub(crate) mod tests { } pub(crate) fn expected_url(name: &str) -> String { - format!("http://127.0.0.1:8081/data/{name}.vcf.gz") + format!("http://127.0.0.1:8081/{name}.vcf.gz") } } diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index 9076de6d..d450425b 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -444,7 +444,7 @@ mod tests { with_local_c4gh_storage(|mut storage| async move { test_range_url( &mut storage, - "http://127.0.0.1:8081/data/folder/key.c4gh", + "http://127.0.0.1:8081/folder/key.c4gh", "folder/key", &Default::default(), ) diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index fac2351f..a1abc69f 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -277,7 +277,7 @@ pub trait UrlFormatter { impl UrlFormatter for storage::file::File { fn format_url>(&self, key: K) -> Result { - let path = Path::new("/").join(self.local_path()).join(key.as_ref()); + let path = Path::new("/").join(key.as_ref()); uri::Builder::new() .scheme(match self.scheme() { Scheme::Http => uri::Scheme::HTTP, @@ -340,7 +340,7 @@ mod tests { fn test_formatter_authority(formatter: storage::file::File, scheme: &str) { assert_eq!( formatter.format_url("path").unwrap(), - format!("{}://127.0.0.1:8080{}/path", scheme, "/data") + format!("{}://127.0.0.1:8080/path", scheme) ) } } diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index b84d68c2..04f0ac8b 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -261,7 +261,7 @@ pub(crate) mod tests { RangeUrlOptions::new_with_default_range(&Default::default()), ) .await; - let expected = Url::new("http://127.0.0.1:8081/data/key1"); + let expected = Url::new("http://127.0.0.1:8081/key1"); assert!(matches!(result, Ok(url) if url == expected)); }) .await; @@ -279,7 +279,7 @@ pub(crate) mod tests { ), ) .await; - let expected = Url::new("http://127.0.0.1:8081/data/key1") + let expected = Url::new("http://127.0.0.1:8081/key1") .with_headers(Headers::default().with_header("Range", "bytes=7-9")); assert!(matches!(result, Ok(url) if url == expected)); }) @@ -295,7 +295,7 @@ pub(crate) mod tests { RangeUrlOptions::new(BytesPosition::new(Some(7), None, None), &Default::default()), ) .await; - let expected = Url::new("http://127.0.0.1:8081/data/key1") + let expected = Url::new("http://127.0.0.1:8081/key1") .with_headers(Headers::default().with_header("Range", "bytes=7-")); assert!(matches!(result, Ok(url) if url == expected)); }) diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index 5be34c5a..8f3dffe7 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -59,9 +59,9 @@ tempfile = { version = "3", optional = true } aws-sdk-s3 = { version = "1", features = ["test-util"], optional = true } aws-config = { version = "1", optional = true } aws-credential-types = { version = "1", features = ["test-util"], optional = true } -s3s = { version = "0.10", optional = true } -s3s-fs = { version = "0.10", optional = true } -s3s-aws = { version = "0.10", optional = true } +s3s = { version = "0.11.0-dev", git = "https://github.com/Nugine/s3s", optional = true } +s3s-fs = { version = "0.11.0-dev", git = "https://github.com/Nugine/s3s", optional = true } +s3s-aws = { version = "0.11.0-dev", git = "https://github.com/Nugine/s3s", optional = true } # Crypt4GH crypt4gh = { version = "0.4", git = "https://github.com/EGA-archive/crypt4gh-rust", optional = true } diff --git a/htsget-test/src/aws_mocks.rs b/htsget-test/src/aws_mocks.rs index f531414f..5a61c80c 100644 --- a/htsget-test/src/aws_mocks.rs +++ b/htsget-test/src/aws_mocks.rs @@ -4,6 +4,7 @@ use aws_credential_types::Credentials; use aws_sdk_s3::config::{BehaviorVersion, Region}; use aws_sdk_s3::Client; use s3s::auth::SimpleAuth; +use s3s::host::SingleDomain; use s3s::service::S3ServiceBuilder; use s3s_fs::FileSystem; use std::future::Future; @@ -32,10 +33,11 @@ pub async fn run_s3_test_server( let fs = FileSystem::new(server_base_path).unwrap(); let auth = SimpleAuth::from_single(cred.access_key_id(), cred.secret_access_key()); + let host = SingleDomain::new(domain_name).unwrap(); let mut service = S3ServiceBuilder::new(fs); service.set_auth(auth); - service.set_base_domain(domain_name); + service.set_host(host); s3s_aws::Client::from(service.build().into_shared()) }; diff --git a/htsget-test/src/http/server.rs b/htsget-test/src/http/server.rs index 818819cd..b9885969 100644 --- a/htsget-test/src/http/server.rs +++ b/htsget-test/src/http/server.rs @@ -329,7 +329,7 @@ where /// An example VCF search response. pub fn expected_response(class: Class, url_path: String) -> Value { - let url = format!("{url_path}/data/vcf/sample1-bcbio-cancer.vcf.gz"); + let url = format!("{url_path}/vcf/sample1-bcbio-cancer.vcf.gz"); let urls = match class { Class::Header => json!([{ From be2b22fe24723f10fc895dba5c9f86d26360776a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 18 Dec 2024 10:04:29 +1100 Subject: [PATCH 11/12] refactor: rename s3-storage to s3 and url-storage to url --- .github/workflows/build.yml | 8 +-- deploy/README.md | 2 +- deploy/bin/htsget-lambda.ts | 4 +- deploy/bin/settings.ts | 4 +- deploy/lib/htsget-lambda-stack.ts | 69 ++++++------------- htsget-actix/Cargo.toml | 4 +- htsget-actix/README.md | 4 +- htsget-axum/Cargo.toml | 20 +++--- htsget-axum/README.md | 12 ++-- htsget-config/Cargo.toml | 6 +- htsget-config/README.md | 12 ++-- .../examples/config-files/s3_storage.toml | 2 +- .../config-files/tls_ticket_server.toml | 2 +- .../examples/config-files/url_storage.toml | 2 +- .../src/config/advanced/allow_guard.rs | 11 ++- htsget-config/src/config/advanced/cors.rs | 4 ++ htsget-config/src/config/advanced/mod.rs | 4 +- .../src/config/advanced/regex_location.rs | 6 +- htsget-config/src/config/advanced/url.rs | 4 ++ htsget-config/src/config/data_server.rs | 1 + htsget-config/src/config/location.rs | 16 +++-- htsget-config/src/config/mod.rs | 13 ++-- htsget-config/src/config/parser.rs | 3 + htsget-config/src/config/ticket_server.rs | 1 + htsget-config/src/error.rs | 3 + htsget-config/src/resolver.rs | 33 ++++----- htsget-config/src/storage/c4gh/local.rs | 4 +- htsget-config/src/storage/c4gh/mod.rs | 9 +-- htsget-config/src/storage/mod.rs | 29 ++++---- htsget-config/src/storage/s3.rs | 4 ++ htsget-config/src/storage/url.rs | 4 ++ htsget-config/src/tls/mod.rs | 2 +- htsget-config/src/types.rs | 31 +++++++++ htsget-http/Cargo.toml | 4 +- htsget-http/README.md | 4 +- htsget-lambda/Cargo.toml | 4 +- htsget-lambda/README.md | 4 +- htsget-search/Cargo.toml | 16 ++--- htsget-search/README.md | 4 +- htsget-search/src/bam_search.rs | 8 +-- htsget-search/src/bcf_search.rs | 8 +-- htsget-search/src/cram_search.rs | 8 +-- htsget-search/src/from_storage.rs | 10 +-- htsget-search/src/lib.rs | 2 +- htsget-search/src/vcf_search.rs | 8 +-- htsget-storage/Cargo.toml | 12 ++-- htsget-storage/README.md | 4 +- htsget-storage/src/c4gh/storage.rs | 28 ++++---- htsget-storage/src/error.rs | 4 +- htsget-storage/src/lib.rs | 14 ++-- htsget-test/Cargo.toml | 6 +- htsget-test/README.md | 4 +- 52 files changed, 264 insertions(+), 221 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71dffe7d..53749167 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: run: cargo build --all-targets --all-features - name: Build no default features run: cargo build --all-targets --no-default-features - - name: Build s3-storage - run: cargo build --all-targets --features s3-storage - - name: Build url-storage - run: cargo build --all-targets --features url-storage + - name: Build s3 + run: cargo build --all-targets --features s3 + - name: Build url + run: cargo build --all-targets --features url diff --git a/deploy/README.md b/deploy/README.md index 6bc1f15a..176eb774 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -29,7 +29,7 @@ These are general settings for the CDK deployment. | `copyTestData?` | Whether to copy test data into the bucket. Defaults to true. This copies the example data under the `data` directory to those buckets. This option only has an affect is `createS3Buckets` is true. | `boolean` | | `copyExampleKeys?` | Whether to create secrets corresponding to C4GH public and private keys that can be used with C4GH storage. This copies the private and public keys in the data directory. Note that private keys copied here are visible in the CDK template. This is not considered secure and should only be used for test data. Real secrets should be manually provisioned or created outside the CDK template. Defaults to false. Secrets are created with [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). | `boolean` | | `secretArns?` | The Secrets Manager secrets which htsget-rs needs access to. This affects the permissions that get added to the Lambda role by policy actions target `secretsmanager:GetSecretValue`. Secrets specified here get added as resources in the policy statement. Permissions are automatically added if `copyExampleKeys` is specified, even if this option is set to `[]`. | `string[]` | -| `features?` | Additional features to compile htsget-rs with. Defaults to `[]`. `s3-storage` is always enabled. | `string[]` | +| `features?` | Additional features to compile htsget-rs with. Defaults to `[]`. `s3` is always enabled. | `string[]` | #### HtsgetJwtAuthSettings diff --git a/deploy/bin/htsget-lambda.ts b/deploy/bin/htsget-lambda.ts index 59bed839..b6bc0c28 100644 --- a/deploy/bin/htsget-lambda.ts +++ b/deploy/bin/htsget-lambda.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import "source-map-support/register"; import * as cdk from "aws-cdk-lib"; -import { HtsgetLambdaStack } from "../lib/htsget-lambda-stack"; -import { SETTINGS } from "./settings"; +import {HtsgetLambdaStack} from "../lib/htsget-lambda-stack"; +import {SETTINGS} from "./settings"; export const STACK_NAME = "HtsgetLambdaStack"; const STACK_DESCRIPTION = "A stack deploying htsget-lambda with API gateway."; diff --git a/deploy/bin/settings.ts b/deploy/bin/settings.ts index 738ce79b..11a4e51a 100644 --- a/deploy/bin/settings.ts +++ b/deploy/bin/settings.ts @@ -1,4 +1,4 @@ -import { HtsgetSettings } from "../lib/htsget-lambda-stack"; +import {HtsgetSettings} from "../lib/htsget-lambda-stack"; /** * Settings to use for the htsget deployment. @@ -20,7 +20,7 @@ export const SETTINGS: HtsgetSettings = { // jwtAudience: ["audience"], // cogUserPoolId: "user-pool-id", }, - // Enable additional features for compiling htsget-rs. `s3-storage` is always enabled. + // Enable additional features for compiling htsget-rs. `s3` is always enabled. features: ["experimental"], copyExampleKeys: true, }; diff --git a/deploy/lib/htsget-lambda-stack.ts b/deploy/lib/htsget-lambda-stack.ts index daee9d3c..29573e84 100644 --- a/deploy/lib/htsget-lambda-stack.ts +++ b/deploy/lib/htsget-lambda-stack.ts @@ -1,49 +1,24 @@ -import { STACK_NAME } from "../bin/htsget-lambda"; +import {STACK_NAME} from "../bin/htsget-lambda"; import * as TOML from "@iarna/toml"; -import { readFileSync } from "fs"; - -import { - CfnOutput, - Duration, - RemovalPolicy, - SecretValue, - Stack, - StackProps, - Tags, -} from "aws-cdk-lib"; -import { Construct } from "constructs"; - -import { UserPool } from "aws-cdk-lib/aws-cognito"; -import { - ManagedPolicy, - PolicyStatement, - Role, - ServicePrincipal, -} from "aws-cdk-lib/aws-iam"; -import { Architecture } from "aws-cdk-lib/aws-lambda"; -import { - Certificate, - CertificateValidation, -} from "aws-cdk-lib/aws-certificatemanager"; -import { ARecord, HostedZone, RecordTarget } from "aws-cdk-lib/aws-route53"; -import { ApiGatewayv2DomainProperties } from "aws-cdk-lib/aws-route53-targets"; -import { RustFunction } from "cargo-lambda-cdk"; +import {readFileSync} from "fs"; + +import {CfnOutput, Duration, RemovalPolicy, SecretValue, Stack, StackProps, Tags,} from "aws-cdk-lib"; +import {Construct} from "constructs"; + +import {UserPool} from "aws-cdk-lib/aws-cognito"; +import {ManagedPolicy, PolicyStatement, Role, ServicePrincipal,} from "aws-cdk-lib/aws-iam"; +import {Architecture} from "aws-cdk-lib/aws-lambda"; +import {Certificate, CertificateValidation,} from "aws-cdk-lib/aws-certificatemanager"; +import {ARecord, HostedZone, RecordTarget} from "aws-cdk-lib/aws-route53"; +import {ApiGatewayv2DomainProperties} from "aws-cdk-lib/aws-route53-targets"; +import {RustFunction} from "cargo-lambda-cdk"; import path from "path"; -import { HttpLambdaIntegration } from "aws-cdk-lib/aws-apigatewayv2-integrations"; -import { - CorsHttpMethod, - DomainName, - HttpApi, - HttpMethod, -} from "aws-cdk-lib/aws-apigatewayv2"; -import { HttpJwtAuthorizer } from "aws-cdk-lib/aws-apigatewayv2-authorizers"; -import { - BlockPublicAccess, - Bucket, - BucketEncryption, -} from "aws-cdk-lib/aws-s3"; -import { BucketDeployment, Source } from "aws-cdk-lib/aws-s3-deployment"; -import { Secret } from "aws-cdk-lib/aws-secretsmanager"; +import {HttpLambdaIntegration} from "aws-cdk-lib/aws-apigatewayv2-integrations"; +import {CorsHttpMethod, DomainName, HttpApi, HttpMethod,} from "aws-cdk-lib/aws-apigatewayv2"; +import {HttpJwtAuthorizer} from "aws-cdk-lib/aws-apigatewayv2-authorizers"; +import {BlockPublicAccess, Bucket, BucketEncryption,} from "aws-cdk-lib/aws-s3"; +import {BucketDeployment, Source} from "aws-cdk-lib/aws-s3-deployment"; +import {Secret} from "aws-cdk-lib/aws-secretsmanager"; /** * Settings related to the htsget lambda stack. @@ -122,7 +97,7 @@ export type HtsgetSettings = { secretArns?: string[]; /** - * Additional features to compile htsget-rs with. Defaults to `[]`. `s3-storage` is always enabled. + * Additional features to compile htsget-rs with. Defaults to `[]`. `s3` is always enabled. */ features?: string[]; }; @@ -274,8 +249,8 @@ export class HtsgetLambdaStack extends Stack { let features = settings.features ?? []; features = features - .filter((f) => f !== "s3-storage") - .concat(["s3-storage"]); + .filter((f) => f !== "s3") + .concat(["s3"]); let htsgetLambda = new RustFunction(this, id + "Function", { manifestPath: path.join(__dirname, "..", ".."), diff --git a/htsget-actix/Cargo.toml b/htsget-actix/Cargo.toml index 1c423995..a0ce09e2 100644 --- a/htsget-actix/Cargo.toml +++ b/htsget-actix/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-actix/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-axum/s3-storage", "htsget-test/s3-storage"] -url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-axum/url-storage", "htsget-test/url-storage"] +s3 = ["htsget-config/s3", "htsget-search/s3", "htsget-http/s3", "htsget-axum/s3", "htsget-test/s3"] +url = ["htsget-config/url", "htsget-search/url", "htsget-http/url", "htsget-axum/url", "htsget-test/url"] experimental = [ "htsget-config/experimental", "htsget-search/experimental", diff --git a/htsget-actix/README.md b/htsget-actix/README.md index c68665fb..d7d49335 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -53,8 +53,8 @@ are exposed in the public API. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Benchmarks diff --git a/htsget-axum/Cargo.toml b/htsget-axum/Cargo.toml index 2eae66fa..56d32d37 100644 --- a/htsget-axum/Cargo.toml +++ b/htsget-axum/Cargo.toml @@ -11,18 +11,18 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-axum/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = [ - "htsget-config/s3-storage", - "htsget-search/s3-storage", - "htsget-test/s3-storage", +s3 = [ + "htsget-config/s3", + "htsget-search/s3", + "htsget-test/s3", "htsget-test/aws-mocks", - "htsget-http/s3-storage" + "htsget-http/s3" ] -url-storage = [ - "htsget-config/url-storage", - "htsget-search/url-storage", - "htsget-test/url-storage", - "htsget-http/url-storage" +url = [ + "htsget-config/url", + "htsget-search/url", + "htsget-test/url", + "htsget-http/url" ] experimental = [ "htsget-config/experimental", diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 0311e9a7..4eea81f8 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -42,14 +42,14 @@ This crate uses [htsget-config] for configuration. Using the default configuration, this will start a ticket server on `127.0.0.1:8080` and a data block server on `127.0.0.1:8081` with data accessible from the [`data`][data] directory. This application supports storage backends defined in [htsget-storage]. -To use `S3Storage`, compile with the `s3-storage` feature: +To use `S3` locations, compile with the `s3` feature: ```sh -cargo run -p htsget-axum --features s3-storage +cargo run -p htsget-axum --features s3 ``` -This will start a ticket server with `S3Storage` using a bucket called `"data"`. +This will start a ticket server with `S3` locations and a bucket called `"data"`. -To use `UrlStorage`, compile with the `url-storage` feature. +To use `Url` locations, compile with the `url` feature. See [htsget-search] for details on how to structure files. @@ -178,8 +178,8 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 5b3a3c1c..b412bbbd 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-config/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["dep:aws-sdk-secretsmanager", "dep:aws-config", "dep:tempfile"] -url-storage = ["dep:reqwest", "dep:cfg-if"] +s3 = ["dep:aws-sdk-secretsmanager", "dep:aws-config", "dep:tempfile"] +url = ["dep:reqwest", "dep:cfg-if"] experimental = ["dep:crypt4gh", "dep:tokio", "dep:futures-util"] default = [] @@ -36,7 +36,7 @@ rustls-pemfile = "2" rustls = "0.23" rustls-pki-types = "1" -# url-storage +# url reqwest = { version = "0.12", features = ["rustls-tls"], default-features = false, optional = true } cfg-if = { version = "1", optional = true } diff --git a/htsget-config/README.md b/htsget-config/README.md index 27555efb..829ff723 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -82,7 +82,7 @@ data_server = "None" > [!IMPORTANT] > Some parts of htsget-rs require extra feature flags for conditional compilation, that's why the examples specify -> using `--all-features`. Notably, `--features s3-storage` enables the `S3` location type, and `--features url-storage` +> using `--all-features`. Notably, `--features s3` enables the `S3` location type, and `--features url` > enabled the remote HTTP server location type. If using a subset of features, for example S3 locations only, then > a single feature can be enabled instead of using `--all-features`. @@ -305,7 +305,7 @@ This project uses [rustls] for all TLS logic, and it does not depend on OpenSSL. strict when accepting certificates and keys. If generating certificates for `root_store` using OpenSSL, the correct extensions, such as `subjectAltName` should be included. -An example of generating a custom root CA and certificates for a `UrlStorage` backend: +An example of generating a custom root CA and certificates for a `Url` backend: ```sh # Create a root CA @@ -314,7 +314,7 @@ openssl req -x509 -noenc -subj '/CN=localhost' -newkey rsa -keyout root.key -out # Create a certificate signing request openssl req -noenc -newkey rsa -keyout server.key -out server.csr -subj '/CN=localhost' -addext subjectAltName=DNS:localhost -# Create the `UrlStorage` server's certificate +# Create the `Url` server's certificate openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -days 365 -out server.crt -copy_extensions copy # An additional client certificate signing request and certificate can be created in the same way as the server @@ -384,7 +384,7 @@ location.keys.private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret location.keys.public = "data/c4gh/keys/alice.pub" ``` -Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `keys.kind = "SecretsManager"` under +Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3` feature flag and specify `keys.kind = "SecretsManager"` under `location` to fetch keys from Secrets Manager. When using Secrets Manager, the `private` and `public` correspond to ARNs or secret names in Secrets Manager storing PEM formatted keys. @@ -442,8 +442,8 @@ regex, and changing it by using a substitution string. Advanced configuration op ### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3` functionality. -* `url-storage`: used to enable `Url` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 7948e457..58c1dd74 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -1,5 +1,5 @@ # An example for a server which uses s3 storage with data located in "bucket". -# Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/s3_storage.toml` +# Run with `cargo run -p htsget-axum --features s3 -- --config htsget-config/examples/config-files/s3_storage.toml` ticket_server.cors.allow_headers = "All" ticket_server.cors.allow_methods = "All" diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index d73bb596..fa99cd8a 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -1,5 +1,5 @@ # An example config file for a TLS ticket server that uses S3 as a storage backend. -# Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/tls_ticket_server.toml` +# Run with `cargo run -p htsget-axum --features s3 -- --config htsget-config/examples/config-files/tls_ticket_server.toml` ticket_server.addr = "0.0.0.0:8080" ticket_server.cors_allow_origins = "All" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index 078a41b4..549a2ea0 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -1,6 +1,6 @@ # An example for a server which uses url storage with data located at "http://localhost:3000". # Run with -# `cargo run -p htsget-axum --features url-storage -- --config htsget-config/examples/config-files/url_storage.toml` +# `cargo run -p htsget-axum --features url -- --config htsget-config/examples/config-files/url_storage.toml` # in the project directory. ticket_server.addr = "127.0.0.1:8082" diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs index 103b8675..1fda3bab 100644 --- a/htsget-config/src/config/advanced/allow_guard.rs +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -1,12 +1,17 @@ //! Allow guard configuration. //! -use crate::resolver::QueryAllowed; use crate::types::Format::{Bam, Bcf, Cram, Vcf}; use crate::types::{Class, Fields, Format, Interval, Query, TaggedTypeAll, Tags}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; +/// Determines whether the query matches for use with the storage. +pub trait QueryAllowed { + /// Does this query match. + fn query_allowed(&self, query: &Query) -> bool; +} + /// A query guard represents query parameters that can be allowed to storage for a given query. #[derive(Serialize, Clone, Debug, Deserialize, PartialEq, Eq)] #[serde(default, deny_unknown_fields)] @@ -145,7 +150,7 @@ impl QueryAllowed for AllowGuard { mod tests { use super::*; use crate::config::tests::test_serialize_and_deserialize; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use crate::config::Config; use crate::types::Class::Header; @@ -260,7 +265,7 @@ mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn allow_guard() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/advanced/cors.rs b/htsget-config/src/config/advanced/cors.rs index 0e8618d7..bebbe9e5 100644 --- a/htsget-config/src/config/advanced/cors.rs +++ b/htsget-config/src/config/advanced/cors.rs @@ -1,3 +1,6 @@ +//! Configuration related to CORS. +//! + use std::fmt::{Display, Formatter}; use std::str::FromStr; @@ -134,6 +137,7 @@ where pub struct HeaderValue(HeaderValueInner); impl HeaderValue { + /// Get the inner header value. pub fn into_inner(self) -> HeaderValueInner { self.0 } diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index 516b93b5..6637dc86 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -1,4 +1,4 @@ -//! Advanced configuration. +//! Configuration options that are advanced in the documentation. //! use serde::{Deserialize, Serialize}; @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; pub mod allow_guard; pub mod cors; pub mod regex_location; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod url; /// Determines which tracing formatting style to use. diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index d8bff741..1d6a72fc 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -98,7 +98,7 @@ mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn regex_location_s3() { test_serialize_and_deserialize( @@ -127,7 +127,7 @@ mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[test] fn regex_location_url() { test_serialize_and_deserialize( @@ -158,7 +158,7 @@ mod tests { ); } - #[cfg(all(feature = "url-storage", feature = "s3-storage"))] + #[cfg(all(feature = "url", feature = "s3"))] #[test] fn regex_location_multiple() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/advanced/url.rs b/htsget-config/src/config/advanced/url.rs index 4fb7ba1e..d1daa3f5 100644 --- a/htsget-config/src/config/advanced/url.rs +++ b/htsget-config/src/config/advanced/url.rs @@ -1,3 +1,6 @@ +//! The config for remote URL server locations. +//! + use crate::error::Error; use crate::error::Error::ParseError; use crate::error::Result; @@ -10,6 +13,7 @@ use http::Uri; use reqwest::Client; use serde::{Deserialize, Serialize}; +/// Options for the remote URL server config. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct Url { diff --git a/htsget-config/src/config/data_server.rs b/htsget-config/src/config/data_server.rs index e9a39447..bf89eff2 100644 --- a/htsget-config/src/config/data_server.rs +++ b/htsget-config/src/config/data_server.rs @@ -83,6 +83,7 @@ impl DataServerConfig { &self.cors } + /// Get the owned TLS config. pub fn into_tls(self) -> Option { self.tls } diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index 508fcdb8..6e870a1e 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -1,4 +1,4 @@ -//! Location configuration. +//! Storage location configuration. //! use crate::config::advanced::regex_location::RegexLocation; @@ -10,7 +10,7 @@ use crate::types::Scheme; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; use std::result; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] use {crate::config::advanced::url::Url, crate::error, http::Uri}; /// The locations of data. @@ -19,18 +19,22 @@ use {crate::config::advanced::url::Url, crate::error, http::Uri}; pub struct Locations(Vec); impl Locations { + /// Create new locations. pub fn new(locations: Vec) -> Self { Self(locations) } + /// Get locations as a slice of `LocationEither`. pub fn as_slice(&self) -> &[LocationEither] { self.0.as_slice() } + /// Get locations as an owned vector of `LocationEither`. pub fn into_inner(self) -> Vec { self.0 } + /// Get locations as a mutable slice of `LocationEither`. pub fn as_mut_slice(&mut self) -> &mut [LocationEither] { self.0.as_mut_slice() } @@ -210,7 +214,7 @@ impl<'de> Deserialize<'de> for StringLocation { }); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] if let Some(s) = s.strip_prefix("s3://") { let (bucket, prefix) = split(s)?; return Ok(StringLocation { @@ -219,7 +223,7 @@ impl<'de> Deserialize<'de> for StringLocation { }); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] if let Some(s_stripped) = s .strip_prefix("http://") .or_else(|| s.strip_prefix("https://")) @@ -326,7 +330,7 @@ mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn location_s3() { test_serialize_and_deserialize( @@ -360,7 +364,7 @@ mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[test] fn location_url() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 9e9d36aa..1e78e5f6 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -1,3 +1,6 @@ +//! Structs to serialize and deserialize the htsget-rs config options. +//! + use std::fmt::Debug; use std::io; use std::path::{Path, PathBuf}; @@ -26,7 +29,7 @@ pub mod parser; pub mod service_info; pub mod ticket_server; -/// Represents a usage string for htsget-rs. +/// The usage string for htsget-rs. pub const USAGE: &str = "To configure htsget-rs use a config file or environment variables. \ See the documentation of the htsget-config crate for more information."; @@ -218,7 +221,7 @@ pub(crate) mod tests { use crate::types::Scheme; use figment::Jail; use http::uri::Authority; - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] use http::Uri; use serde::de::DeserializeOwned; use serde_json::json; @@ -569,7 +572,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn simple_locations_s3() { test_config_from_file( @@ -587,7 +590,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[test] fn simple_locations_url() { test_config_from_file( @@ -618,7 +621,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn simple_locations_multiple_mixed() { test_config_from_file( diff --git a/htsget-config/src/config/parser.rs b/htsget-config/src/config/parser.rs index 721ece3d..77886921 100644 --- a/htsget-config/src/config/parser.rs +++ b/htsget-config/src/config/parser.rs @@ -1,3 +1,6 @@ +//! Parse config for a file and environment variables. +//! + use crate::config::Config; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; diff --git a/htsget-config/src/config/ticket_server.rs b/htsget-config/src/config/ticket_server.rs index 3cc659a0..e23b8643 100644 --- a/htsget-config/src/config/ticket_server.rs +++ b/htsget-config/src/config/ticket_server.rs @@ -37,6 +37,7 @@ impl TicketServerConfig { &self.cors } + /// Get the owned TLS config. pub fn into_tls(self) -> Option { self.tls } diff --git a/htsget-config/src/error.rs b/htsget-config/src/error.rs index 50eba8de..a6d89ac4 100644 --- a/htsget-config/src/error.rs +++ b/htsget-config/src/error.rs @@ -1,3 +1,6 @@ +//! Error types used by this crate. +//! + use std::{io, result}; use thiserror::Error; diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index 1f084b7b..c1b1f5f4 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -1,3 +1,6 @@ +//! Resolvers map ids to storage locations. + +use crate::config::advanced::allow_guard::QueryAllowed; use crate::config::advanced::regex_location::RegexLocation; use crate::config::location::{LocationEither, Locations}; use crate::storage; @@ -20,11 +23,11 @@ pub trait ResolveResponse { async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result; /// Convert from `S3`. - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result; /// Convert from `Url`. - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] async fn from_url(url_storage: &storage::url::Url, query: &Query) -> Result; } @@ -38,12 +41,6 @@ pub trait StorageResolver { ) -> Option>; } -/// Determines whether the query matches for use with the storage. -pub trait QueryAllowed { - /// Does this query match. - fn query_allowed(&self, query: &Query) -> bool; -} - /// A type which holds a resolved storage and an resolved id. #[derive(Debug)] pub struct ResolvedStorage { @@ -125,7 +122,7 @@ impl StorageResolver for LocationEither { match self.backend() { Backend::File(file) => Some(T::from_file(file, query).await), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] Backend::S3(s3) => { let s3 = if let Self::Regex(regex_location) = self { if s3.bucket().is_empty() { @@ -145,7 +142,7 @@ impl StorageResolver for LocationEither { Some(T::from_s3(s3, query).await) } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] Backend::Url(url_storage) => Some(T::from_url(url_storage, query).await), } } @@ -203,9 +200,9 @@ mod tests { use crate::types::Scheme::Http; use crate::types::Url; use http::uri::Authority; - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] use reqwest::ClientBuilder; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use { crate::config::advanced::allow_guard::{AllowGuard, ReferenceNames}, crate::types::{Class, Fields, Interval, Tags}, @@ -223,7 +220,7 @@ mod tests { )) } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { Ok(Response::new( Bam, @@ -231,7 +228,7 @@ mod tests { )) } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] async fn from_url(url: &storage::url::Url, query: &Query) -> Result { Ok(Response::new( Bam, @@ -266,7 +263,7 @@ mod tests { expected_resolved_request(vec![location.into()], "127.0.0.1:8080/id-1").await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { let s3_storage = storage::s3::S3::new("id2".to_string(), None, false); @@ -282,7 +279,7 @@ mod tests { expected_resolved_request(vec![location.into()], "id2/id-1").await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn resolver_resolve_s3_request() { let regex_location = RegexLocation::new( @@ -308,7 +305,7 @@ mod tests { expected_resolved_request(vec![location.into()], "bucket/id-1").await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn resolver_resolve_url_request() { let client = ClientBuilder::new().build().unwrap(); @@ -443,7 +440,7 @@ mod tests { }); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn config_resolvers_all_options_env() { test_config_from_env( diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index 696abc8b..116343e6 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -98,7 +98,7 @@ mod tests { }); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn config_s3_storage_c4gh() { test_c4gh_storage_config( @@ -115,7 +115,7 @@ mod tests { ); } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn config_url_storage_c4gh() { test_c4gh_storage_config( diff --git a/htsget-config/src/storage/c4gh/mod.rs b/htsget-config/src/storage/c4gh/mod.rs index 260ef4f5..9f5ab17d 100644 --- a/htsget-config/src/storage/c4gh/mod.rs +++ b/htsget-config/src/storage/c4gh/mod.rs @@ -4,7 +4,7 @@ use crate::error::Error::{IoError, ParseError}; use crate::error::{Error, Result}; use crate::storage::c4gh::local::C4GHLocal; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "s3")] use crate::storage::c4gh::secrets_manager::C4GHSecretsManager; use crypt4gh::error::Crypt4GHError; use futures_util::future::{BoxFuture, Shared}; @@ -14,7 +14,7 @@ use tokio::task::{JoinError, JoinHandle}; pub mod local; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "s3")] pub mod secrets_manager; /// Config for Crypt4GH keys. @@ -40,6 +40,7 @@ impl C4GHKeys { }] } + /// Construct from an existing join handle. pub fn from_join_handle(handle: JoinHandle>>) -> Self { Self { keys: handle.map(|value| value?).boxed().shared(), @@ -65,7 +66,7 @@ impl TryFrom for C4GHKeys { fn try_from(location: C4GHKeyLocation) -> Result { match location { C4GHKeyLocation::File(file) => file.try_into(), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] C4GHKeyLocation::SecretsManager(secrets_manager) => secrets_manager.try_into(), } } @@ -78,7 +79,7 @@ impl TryFrom for C4GHKeys { pub enum C4GHKeyLocation { #[serde(alias = "file", alias = "FILE")] File(C4GHLocal), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[serde(alias = "secretsmanager", alias = "SECRETSMANAGER")] SecretsManager(C4GHSecretsManager), } diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index 74f10b7d..25c3f521 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,19 +1,22 @@ -#[cfg(any(feature = "url-storage", feature = "s3-storage"))] +//! Storage backends. +//! + +#[cfg(any(feature = "url", feature = "s3"))] use crate::error::Error; use crate::error::Result; use crate::storage::file::File; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "s3")] use crate::storage::s3::S3; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] use crate::storage::url::Url; use serde::{Deserialize, Serialize}; #[cfg(feature = "experimental")] pub mod c4gh; pub mod file; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "s3")] pub mod s3; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod url; /// A new type representing a resolved id. @@ -39,10 +42,10 @@ impl ResolvedId { pub enum Backend { #[serde(alias = "file", alias = "FILE")] File(File), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[serde(alias = "s3")] S3(S3), - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[serde(alias = "url", alias = "URL")] Url(Url), } @@ -52,15 +55,15 @@ impl Backend { pub fn as_file(&self) -> Result<&File> { match self { Backend::File(file) => Ok(file), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] Backend::S3(_) => Err(Error::ParseError("not a `File` variant".to_string())), - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] Backend::Url(_) => Err(Error::ParseError("not a `File` variant".to_string())), } } /// Get the file variant and error if it is not `S3`. - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] pub fn as_s3(&self) -> Result<&S3> { if let Backend::S3(s3) = self { Ok(s3) @@ -70,7 +73,7 @@ impl Backend { } /// Get the url variant and error if it is not `Url`. - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] pub fn as_url(&self) -> Result<&Url> { if let Backend::Url(url) = self { Ok(url) @@ -121,7 +124,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn config_storage_tagged_s3_file() { test_config_from_file( @@ -139,7 +142,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[test] fn config_storage_tagged_s3_env() { test_config_from_env( diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index ea22d2b6..7f18e218 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -1,7 +1,11 @@ +//! Configuration for storage on AWS S3. +//! + #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; use serde::{Deserialize, Serialize}; +/// Configuration struct for S3 storage. #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default, deny_unknown_fields)] pub struct S3 { diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index f576dc4a..2f9a9928 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -1,3 +1,6 @@ +//! Configuration for remote URL server storage. +//! + use crate::config::advanced; #[cfg(feature = "experimental")] use crate::storage::c4gh::C4GHKeys; @@ -5,6 +8,7 @@ use http::Uri; use reqwest::Client; use serde::{Deserialize, Serialize}; +/// Remote URL server storage struct. #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(try_from = "advanced::url::Url", deny_unknown_fields)] pub struct Url { diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs index c1094cce..7518ea48 100644 --- a/htsget-config/src/tls/mod.rs +++ b/htsget-config/src/tls/mod.rs @@ -1,7 +1,7 @@ //! Configuration related to TLS. //! -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod client; use std::fs::File; diff --git a/htsget-config/src/types.rs b/htsget-config/src/types.rs index b1cbc856..171d915c 100644 --- a/htsget-config/src/types.rs +++ b/htsget-config/src/types.rs @@ -1,3 +1,6 @@ +//! Types related to htsget like formats, reference names, classes or intervals. +//! + use std::collections::{HashMap, HashSet}; use std::fmt::{Debug, Display, Formatter}; use std::io::ErrorKind::Other; @@ -13,6 +16,7 @@ use tracing::instrument; use crate::error::Error; use crate::error::Error::ParseError; +/// The result type returning a `HtsGetError`. pub type Result = result::Result; /// An enumeration with all the possible formats. @@ -31,6 +35,7 @@ pub enum Format { /// Todo allow these to be configurable. impl Format { + /// Get the file ending for the format. pub fn file_ending(&self) -> &str { match self { Format::Bam => ".bam", @@ -40,10 +45,12 @@ impl Format { } } + /// Get the file name including its ending. pub fn fmt_file(&self, id: &str) -> String { format!("{id}{}", self.file_ending()) } + /// Get the index file ending for this format. pub fn index_file_ending(&self) -> &str { match self { Format::Bam => ".bam.bai", @@ -53,10 +60,12 @@ impl Format { } } + /// Get the index file name including its ending. pub fn fmt_index(&self, id: &str) -> String { format!("{id}{}", self.index_file_ending()) } + /// Get the GZI index file ending for this format. pub fn gzi_index_file_ending(&self) -> io::Result<&str> { match self { Format::Bam => Ok(".bam.gzi"), @@ -69,6 +78,7 @@ impl Format { } } + /// Get the GZI index file name including its ending. pub fn fmt_gzi(&self, id: &str) -> io::Result { Ok(format!("{id}{}", self.gzi_index_file_ending()?)) } @@ -178,10 +188,12 @@ impl Interval { }) } + /// Start position. pub fn start(&self) -> Option { self.start } + /// End position. pub fn end(&self) -> Option { self.end } @@ -383,43 +395,53 @@ impl Query { self } + /// Id. pub fn id(&self) -> &str { &self.id } + /// Format. pub fn format(&self) -> Format { self.format } + /// Class. pub fn class(&self) -> Class { self.class } + /// Reference name. pub fn reference_name(&self) -> Option<&str> { self.reference_name.as_deref() } + /// Interval. pub fn interval(&self) -> Interval { self.interval } + /// Fields. pub fn fields(&self) -> &Fields { &self.fields } + /// Tags. pub fn tags(&self) -> &Tags { &self.tags } + /// No tags. pub fn no_tags(&self) -> &NoTags { &self.no_tags } + /// Request. pub fn request(&self) -> &Request { &self.request } } +/// Htsget specific errors. #[derive(Error, Debug, PartialEq, Eq)] pub enum HtsGetError { #[error("not found: {0}")] @@ -445,30 +467,37 @@ pub enum HtsGetError { } impl HtsGetError { + /// Create a `NotFound` error. pub fn not_found>(message: S) -> Self { Self::NotFound(message.into()) } + /// Create an `UnsupportedFormat` error. pub fn unsupported_format>(format: S) -> Self { Self::UnsupportedFormat(format.into()) } + /// Create an `InvalidInput` error. pub fn invalid_input>(message: S) -> Self { Self::InvalidInput(message.into()) } + /// Create an `InvalidRange` error. pub fn invalid_range>(message: S) -> Self { Self::InvalidRange(message.into()) } + /// Create an `IoError` error. pub fn io_error>(message: S) -> Self { Self::IoError(message.into()) } + /// Create a `ParseError` error. pub fn parse_error>(message: S) -> Self { Self::ParseError(message.into()) } + /// Create an `InternalError` error. pub fn internal_error>(message: S) -> Self { Self::InternalError(message.into()) } @@ -612,6 +641,7 @@ pub struct JsonResponse { } impl JsonResponse { + /// Create a new `JsonResponse`. pub fn new(htsget: Response) -> Self { Self { htsget } } @@ -631,6 +661,7 @@ pub struct Response { } impl Response { + /// Create a new `Response`. pub fn new(format: Format, urls: Vec) -> Self { Self { format, urls } } diff --git a/htsget-http/Cargo.toml b/htsget-http/Cargo.toml index 6603f12f..a2df23c9 100644 --- a/htsget-http/Cargo.toml +++ b/htsget-http/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-http/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-test/s3-storage"] -url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-test/url-storage"] +s3 = ["htsget-config/s3", "htsget-search/s3", "htsget-test/s3"] +url = ["htsget-config/url", "htsget-search/url", "htsget-test/url"] experimental = ["htsget-config/experimental", "htsget-search/experimental", "htsget-test/experimental"] default = [] diff --git a/htsget-http/README.md b/htsget-http/README.md index b31a5f44..5a8cd66a 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -35,8 +35,8 @@ and process it using [htsget-search] to return JSON HTTP responses. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp diff --git a/htsget-lambda/Cargo.toml b/htsget-lambda/Cargo.toml index 8f3b479a..ad09ddce 100644 --- a/htsget-lambda/Cargo.toml +++ b/htsget-lambda/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-lambda/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = ["htsget-axum/s3-storage", "htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-test/s3-storage"] -url-storage = ["htsget-axum/url-storage", "htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-test/url-storage"] +s3 = ["htsget-axum/s3", "htsget-config/s3", "htsget-search/s3", "htsget-http/s3", "htsget-test/s3"] +url = ["htsget-axum/url", "htsget-config/url", "htsget-search/url", "htsget-http/url", "htsget-test/url"] experimental = [ "htsget-axum/experimental", "htsget-config/experimental", diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index d1058763..d2322781 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -42,8 +42,8 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index 9388ef06..2c72092f 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -11,16 +11,16 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-search/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = [ - "htsget-storage/s3-storage", - "htsget-config/s3-storage", - "htsget-test/s3-storage", +s3 = [ + "htsget-storage/s3", + "htsget-config/s3", + "htsget-test/s3", "htsget-test/aws-mocks" ] -url-storage = [ - "htsget-storage/url-storage", - "htsget-config/url-storage", - "htsget-test/url-storage" +url = [ + "htsget-storage/url", + "htsget-config/url", + "htsget-test/url" ] experimental = [ "htsget-storage/experimental", diff --git a/htsget-search/README.md b/htsget-search/README.md index a849a3d4..683e5c18 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -53,8 +53,8 @@ used to process requests. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 4d80587a..e0a78e5a 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -147,7 +147,7 @@ impl BamSearch { #[cfg(test)] pub(crate) mod tests { use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Body, Class::Header, Headers, HtsGetError::NotFound, Response, Url}; @@ -535,7 +535,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -553,7 +553,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -572,7 +572,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index d4431839..3a17412a 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -113,7 +113,7 @@ mod tests { use std::future::Future; use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::search::SearchAll; @@ -328,7 +328,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -346,7 +346,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -365,7 +365,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index 86121085..6400f38b 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -277,7 +277,7 @@ mod tests { use htsget_test::http::concat::ConcatResponse; use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Header, Headers, HtsGetError::NotFound, Response, Url}; @@ -542,7 +542,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -560,7 +560,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -579,7 +579,7 @@ mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index ca72580b..37d13c99 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -56,14 +56,14 @@ impl ResolveResponse for HtsGetFromStorage { searcher.search(query.clone()).await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { let storage = Storage::from_s3(s3_storage).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] async fn from_url(url_storage_config: &storage::url::Url, query: &Query) -> Result { let storage = Storage::from_url(url_storage_config).await; let searcher = HtsGetFromStorage::new(storage?); @@ -90,7 +90,7 @@ pub(crate) mod tests { use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use { htsget_storage::s3::S3Storage, htsget_test::aws_mocks::with_s3_test_server, std::fs::create_dir, }; @@ -318,7 +318,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] pub(crate) async fn with_aws_storage_fn(test: F, path: &str, copy_files: &[&str]) where F: FnOnce(Storage) -> Fut, @@ -341,7 +341,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] pub(crate) async fn with_aws_s3_storage_fn(test: F, folder_name: String, base_path: &Path) where F: FnOnce(Storage) -> Fut, diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 59d316dd..620ecb5e 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -4,7 +4,7 @@ //! pub use htsget_config::config::Config; -pub use htsget_config::resolver::{IdResolver, QueryAllowed, ResolveResponse, StorageResolver}; +pub use htsget_config::resolver::{IdResolver, ResolveResponse, StorageResolver}; pub use htsget_config::types::{ Class, Format, Headers, HtsGetError, JsonResponse, Query, Response, Result, Url, }; diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index d83209e1..5ac5c8bd 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -124,7 +124,7 @@ pub(crate) mod tests { use std::future::Future; use super::*; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::search::SearchAll; @@ -339,7 +339,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -357,7 +357,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -376,7 +376,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-storage/Cargo.toml b/htsget-storage/Cargo.toml index a9d914bc..89f92eed 100644 --- a/htsget-storage/Cargo.toml +++ b/htsget-storage/Cargo.toml @@ -11,19 +11,19 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-storage/README.m repository = "https://github.com/umccr/htsget-rs" [features] -s3-storage = [ +s3 = [ "dep:bytes", "dep:aws-sdk-s3", "dep:aws-config", - "htsget-config/s3-storage", - "htsget-test/s3-storage", + "htsget-config/s3", + "htsget-test/s3", "htsget-test/aws-mocks" ] -url-storage = [ +url = [ "dep:bytes", "dep:reqwest", - "htsget-config/url-storage", - "htsget-test/url-storage" + "htsget-config/url", + "htsget-test/url" ] experimental = ["dep:crypt4gh", "dep:bincode", "htsget-config/experimental", "htsget-test/experimental"] default = [] diff --git a/htsget-storage/README.md b/htsget-storage/README.md index 1e9a8c53..a606047b 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -46,8 +46,8 @@ and [url] modules implement the `Storage` functionality. #### Feature flags This crate has the following features: -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [local]: src/local.rs diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index d450425b..6063b276 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -315,9 +315,9 @@ impl From for StorageError { mod tests { use super::*; use crate::local::tests::with_local_storage; - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] use crate::s3::tests::with_aws_s3_storage; - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] use crate::url::tests::{test_headers, with_url_test_server}; use htsget_config::types::Headers; use htsget_test::c4gh::{encrypt_data, get_decryption_keys}; @@ -335,7 +335,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn test_preprocess_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -344,7 +344,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_preprocess_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -363,7 +363,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn test_get_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -372,7 +372,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_get_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -391,7 +391,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn test_head_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -400,7 +400,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_head_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -419,7 +419,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn test_postprocess_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -428,7 +428,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_postprocess_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -453,7 +453,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[tokio::test] async fn test_range_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -468,7 +468,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] #[tokio::test] async fn test_range_url_storage() { with_url_c4gh_storage(|mut storage, url| async move { @@ -608,7 +608,7 @@ mod tests { .await; } - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] pub(crate) async fn with_s3_c4gh_storage(test: F) where F: FnOnce(C4GHStorage) -> Fut, @@ -621,7 +621,7 @@ mod tests { .await; } - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] pub(crate) async fn with_url_c4gh_storage(test: F) where F: FnOnce(C4GHStorage, String) -> Fut, diff --git a/htsget-storage/src/error.rs b/htsget-storage/src/error.rs index 2f02d788..cce3e3a2 100644 --- a/htsget-storage/src/error.rs +++ b/htsget-storage/src/error.rs @@ -40,7 +40,7 @@ pub enum StorageError { #[error("response error: {0}")] ResponseError(String), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] #[error("aws error: {0}, with key: `{1}`")] AwsS3Error(String, String), @@ -60,7 +60,7 @@ impl From for HtsGetError { | StorageError::InvalidUri(_) | StorageError::InvalidAddress(_) | StorageError::InternalError(_)) => Self::InternalError(err.to_string()), - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] err @ StorageError::AwsS3Error(_, _) => Self::IoError(err.to_string()), err @ StorageError::UrlParseError(_) => Self::ParseError(err.to_string()), } diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index a1abc69f..eead16e0 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -1,7 +1,7 @@ //! Module providing the abstractions needed to read files from an storage //! -pub use htsget_config::resolver::{IdResolver, QueryAllowed, ResolveResponse, StorageResolver}; +pub use htsget_config::resolver::{IdResolver, ResolveResponse, StorageResolver}; pub use htsget_config::types::{ Class, Format, Headers, HtsGetError, JsonResponse, Query, Response, Url, }; @@ -12,10 +12,10 @@ use crate::error::Result; use crate::error::StorageError; use crate::error::StorageError::InvalidKey; use crate::local::FileStorage; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "s3")] use crate::s3::S3Storage; use crate::types::{BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions}; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] use crate::url::UrlStorage; use async_trait::async_trait; use base64::engine::general_purpose; @@ -38,10 +38,10 @@ use tokio::io::{AsyncRead, ReadBuf}; pub mod c4gh; pub mod error; pub mod local; -#[cfg(feature = "s3-storage")] +#[cfg(feature = "s3")] pub mod s3; pub mod types; -#[cfg(feature = "url-storage")] +#[cfg(feature = "url")] pub mod url; pin_project! { @@ -163,7 +163,7 @@ impl Storage { } /// Create from s3 config. - #[cfg(feature = "s3-storage")] + #[cfg(feature = "s3")] pub async fn from_s3(s3: &storage::s3::S3) -> Result { let storage = Storage::new( S3Storage::new_with_default_config( @@ -184,7 +184,7 @@ impl Storage { } /// Create from url config. - #[cfg(feature = "url-storage")] + #[cfg(feature = "url")] pub async fn from_url(url: &storage::url::Url) -> Result { let storage = Storage::new(UrlStorage::new( url.client_cloned(), diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index 8f3dffe7..c22663da 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -25,7 +25,7 @@ http = [ "dep:base64" ] aws-mocks = [ - "s3-storage", + "s3", "dep:tempfile", "dep:aws-sdk-s3", "dep:aws-config", @@ -34,8 +34,8 @@ aws-mocks = [ "dep:s3s-fs", "dep:s3s-aws" ] -s3-storage = ["htsget-config?/s3-storage"] -url-storage = ["htsget-config?/url-storage"] +s3 = ["htsget-config?/s3"] +url = ["htsget-config?/url"] experimental = ["dep:crypt4gh", "dep:htsget-config", "htsget-config/experimental"] default = [] diff --git a/htsget-test/README.md b/htsget-test/README.md index fd8e2f6c..14441a20 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -34,8 +34,8 @@ This library is intended to be used as a [development dependency][dev-dependenci This crate has the following features: * `http`: used to enable common functionality for HTTP tests. * `aws-mocks`: used to enable AWS mocking for tests. -* `s3-storage`: used to enable `S3Storage` functionality. -* `url-storage`: used to enable `UrlStorage` functionality. +* `s3`: used to enable `S3` location functionality. +* `url`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies From c13d498093467fe94900f610042ad8d47ad6e8ad Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 18 Dec 2024 10:36:25 +1100 Subject: [PATCH 12/12] refactor: revert changes to s3 and url storage feature flags --- .github/workflows/build.yml | 8 +-- deploy/README.md | 2 +- deploy/bin/htsget-lambda.ts | 4 +- deploy/bin/settings.ts | 4 +- deploy/lib/htsget-lambda-stack.ts | 69 +++++++++++++------ htsget-actix/Cargo.toml | 4 +- htsget-actix/README.md | 4 +- htsget-axum/Cargo.toml | 20 +++--- htsget-axum/README.md | 12 ++-- htsget-config/Cargo.toml | 4 +- htsget-config/README.md | 8 +-- .../examples/config-files/s3_storage.toml | 2 +- .../config-files/tls_ticket_server.toml | 2 +- .../examples/config-files/url_storage.toml | 2 +- .../src/config/advanced/allow_guard.rs | 4 +- htsget-config/src/config/advanced/mod.rs | 2 +- .../src/config/advanced/regex_location.rs | 6 +- htsget-config/src/config/location.rs | 10 +-- htsget-config/src/config/mod.rs | 8 +-- htsget-config/src/resolver.rs | 24 +++---- htsget-config/src/storage/c4gh/local.rs | 4 +- htsget-config/src/storage/c4gh/mod.rs | 8 +-- htsget-config/src/storage/mod.rs | 26 +++---- htsget-config/src/tls/mod.rs | 2 +- htsget-http/Cargo.toml | 4 +- htsget-http/README.md | 4 +- htsget-lambda/Cargo.toml | 4 +- htsget-search/Cargo.toml | 16 ++--- htsget-search/README.md | 4 +- htsget-search/src/bam_search.rs | 8 +-- htsget-search/src/bcf_search.rs | 8 +-- htsget-search/src/cram_search.rs | 8 +-- htsget-search/src/from_storage.rs | 10 +-- htsget-search/src/vcf_search.rs | 8 +-- htsget-storage/Cargo.toml | 12 ++-- htsget-storage/README.md | 4 +- htsget-storage/src/c4gh/storage.rs | 28 ++++---- htsget-storage/src/error.rs | 4 +- htsget-storage/src/lib.rs | 12 ++-- htsget-test/Cargo.toml | 6 +- htsget-test/README.md | 4 +- 41 files changed, 204 insertions(+), 179 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53749167..71dffe7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: run: cargo build --all-targets --all-features - name: Build no default features run: cargo build --all-targets --no-default-features - - name: Build s3 - run: cargo build --all-targets --features s3 - - name: Build url - run: cargo build --all-targets --features url + - name: Build s3-storage + run: cargo build --all-targets --features s3-storage + - name: Build url-storage + run: cargo build --all-targets --features url-storage diff --git a/deploy/README.md b/deploy/README.md index 176eb774..6bc1f15a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -29,7 +29,7 @@ These are general settings for the CDK deployment. | `copyTestData?` | Whether to copy test data into the bucket. Defaults to true. This copies the example data under the `data` directory to those buckets. This option only has an affect is `createS3Buckets` is true. | `boolean` | | `copyExampleKeys?` | Whether to create secrets corresponding to C4GH public and private keys that can be used with C4GH storage. This copies the private and public keys in the data directory. Note that private keys copied here are visible in the CDK template. This is not considered secure and should only be used for test data. Real secrets should be manually provisioned or created outside the CDK template. Defaults to false. Secrets are created with [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). | `boolean` | | `secretArns?` | The Secrets Manager secrets which htsget-rs needs access to. This affects the permissions that get added to the Lambda role by policy actions target `secretsmanager:GetSecretValue`. Secrets specified here get added as resources in the policy statement. Permissions are automatically added if `copyExampleKeys` is specified, even if this option is set to `[]`. | `string[]` | -| `features?` | Additional features to compile htsget-rs with. Defaults to `[]`. `s3` is always enabled. | `string[]` | +| `features?` | Additional features to compile htsget-rs with. Defaults to `[]`. `s3-storage` is always enabled. | `string[]` | #### HtsgetJwtAuthSettings diff --git a/deploy/bin/htsget-lambda.ts b/deploy/bin/htsget-lambda.ts index b6bc0c28..59bed839 100644 --- a/deploy/bin/htsget-lambda.ts +++ b/deploy/bin/htsget-lambda.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import "source-map-support/register"; import * as cdk from "aws-cdk-lib"; -import {HtsgetLambdaStack} from "../lib/htsget-lambda-stack"; -import {SETTINGS} from "./settings"; +import { HtsgetLambdaStack } from "../lib/htsget-lambda-stack"; +import { SETTINGS } from "./settings"; export const STACK_NAME = "HtsgetLambdaStack"; const STACK_DESCRIPTION = "A stack deploying htsget-lambda with API gateway."; diff --git a/deploy/bin/settings.ts b/deploy/bin/settings.ts index 11a4e51a..738ce79b 100644 --- a/deploy/bin/settings.ts +++ b/deploy/bin/settings.ts @@ -1,4 +1,4 @@ -import {HtsgetSettings} from "../lib/htsget-lambda-stack"; +import { HtsgetSettings } from "../lib/htsget-lambda-stack"; /** * Settings to use for the htsget deployment. @@ -20,7 +20,7 @@ export const SETTINGS: HtsgetSettings = { // jwtAudience: ["audience"], // cogUserPoolId: "user-pool-id", }, - // Enable additional features for compiling htsget-rs. `s3` is always enabled. + // Enable additional features for compiling htsget-rs. `s3-storage` is always enabled. features: ["experimental"], copyExampleKeys: true, }; diff --git a/deploy/lib/htsget-lambda-stack.ts b/deploy/lib/htsget-lambda-stack.ts index 29573e84..daee9d3c 100644 --- a/deploy/lib/htsget-lambda-stack.ts +++ b/deploy/lib/htsget-lambda-stack.ts @@ -1,24 +1,49 @@ -import {STACK_NAME} from "../bin/htsget-lambda"; +import { STACK_NAME } from "../bin/htsget-lambda"; import * as TOML from "@iarna/toml"; -import {readFileSync} from "fs"; - -import {CfnOutput, Duration, RemovalPolicy, SecretValue, Stack, StackProps, Tags,} from "aws-cdk-lib"; -import {Construct} from "constructs"; - -import {UserPool} from "aws-cdk-lib/aws-cognito"; -import {ManagedPolicy, PolicyStatement, Role, ServicePrincipal,} from "aws-cdk-lib/aws-iam"; -import {Architecture} from "aws-cdk-lib/aws-lambda"; -import {Certificate, CertificateValidation,} from "aws-cdk-lib/aws-certificatemanager"; -import {ARecord, HostedZone, RecordTarget} from "aws-cdk-lib/aws-route53"; -import {ApiGatewayv2DomainProperties} from "aws-cdk-lib/aws-route53-targets"; -import {RustFunction} from "cargo-lambda-cdk"; +import { readFileSync } from "fs"; + +import { + CfnOutput, + Duration, + RemovalPolicy, + SecretValue, + Stack, + StackProps, + Tags, +} from "aws-cdk-lib"; +import { Construct } from "constructs"; + +import { UserPool } from "aws-cdk-lib/aws-cognito"; +import { + ManagedPolicy, + PolicyStatement, + Role, + ServicePrincipal, +} from "aws-cdk-lib/aws-iam"; +import { Architecture } from "aws-cdk-lib/aws-lambda"; +import { + Certificate, + CertificateValidation, +} from "aws-cdk-lib/aws-certificatemanager"; +import { ARecord, HostedZone, RecordTarget } from "aws-cdk-lib/aws-route53"; +import { ApiGatewayv2DomainProperties } from "aws-cdk-lib/aws-route53-targets"; +import { RustFunction } from "cargo-lambda-cdk"; import path from "path"; -import {HttpLambdaIntegration} from "aws-cdk-lib/aws-apigatewayv2-integrations"; -import {CorsHttpMethod, DomainName, HttpApi, HttpMethod,} from "aws-cdk-lib/aws-apigatewayv2"; -import {HttpJwtAuthorizer} from "aws-cdk-lib/aws-apigatewayv2-authorizers"; -import {BlockPublicAccess, Bucket, BucketEncryption,} from "aws-cdk-lib/aws-s3"; -import {BucketDeployment, Source} from "aws-cdk-lib/aws-s3-deployment"; -import {Secret} from "aws-cdk-lib/aws-secretsmanager"; +import { HttpLambdaIntegration } from "aws-cdk-lib/aws-apigatewayv2-integrations"; +import { + CorsHttpMethod, + DomainName, + HttpApi, + HttpMethod, +} from "aws-cdk-lib/aws-apigatewayv2"; +import { HttpJwtAuthorizer } from "aws-cdk-lib/aws-apigatewayv2-authorizers"; +import { + BlockPublicAccess, + Bucket, + BucketEncryption, +} from "aws-cdk-lib/aws-s3"; +import { BucketDeployment, Source } from "aws-cdk-lib/aws-s3-deployment"; +import { Secret } from "aws-cdk-lib/aws-secretsmanager"; /** * Settings related to the htsget lambda stack. @@ -97,7 +122,7 @@ export type HtsgetSettings = { secretArns?: string[]; /** - * Additional features to compile htsget-rs with. Defaults to `[]`. `s3` is always enabled. + * Additional features to compile htsget-rs with. Defaults to `[]`. `s3-storage` is always enabled. */ features?: string[]; }; @@ -249,8 +274,8 @@ export class HtsgetLambdaStack extends Stack { let features = settings.features ?? []; features = features - .filter((f) => f !== "s3") - .concat(["s3"]); + .filter((f) => f !== "s3-storage") + .concat(["s3-storage"]); let htsgetLambda = new RustFunction(this, id + "Function", { manifestPath: path.join(__dirname, "..", ".."), diff --git a/htsget-actix/Cargo.toml b/htsget-actix/Cargo.toml index a0ce09e2..1c423995 100644 --- a/htsget-actix/Cargo.toml +++ b/htsget-actix/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-actix/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3 = ["htsget-config/s3", "htsget-search/s3", "htsget-http/s3", "htsget-axum/s3", "htsget-test/s3"] -url = ["htsget-config/url", "htsget-search/url", "htsget-http/url", "htsget-axum/url", "htsget-test/url"] +s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-axum/s3-storage", "htsget-test/s3-storage"] +url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-axum/url-storage", "htsget-test/url-storage"] experimental = [ "htsget-config/experimental", "htsget-search/experimental", diff --git a/htsget-actix/README.md b/htsget-actix/README.md index d7d49335..826c5221 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -53,8 +53,8 @@ are exposed in the public API. #### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Benchmarks diff --git a/htsget-axum/Cargo.toml b/htsget-axum/Cargo.toml index 56d32d37..2eae66fa 100644 --- a/htsget-axum/Cargo.toml +++ b/htsget-axum/Cargo.toml @@ -11,18 +11,18 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-axum/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3 = [ - "htsget-config/s3", - "htsget-search/s3", - "htsget-test/s3", +s3-storage = [ + "htsget-config/s3-storage", + "htsget-search/s3-storage", + "htsget-test/s3-storage", "htsget-test/aws-mocks", - "htsget-http/s3" + "htsget-http/s3-storage" ] -url = [ - "htsget-config/url", - "htsget-search/url", - "htsget-test/url", - "htsget-http/url" +url-storage = [ + "htsget-config/url-storage", + "htsget-search/url-storage", + "htsget-test/url-storage", + "htsget-http/url-storage" ] experimental = [ "htsget-config/experimental", diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 4eea81f8..886e3f1d 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -42,14 +42,14 @@ This crate uses [htsget-config] for configuration. Using the default configuration, this will start a ticket server on `127.0.0.1:8080` and a data block server on `127.0.0.1:8081` with data accessible from the [`data`][data] directory. This application supports storage backends defined in [htsget-storage]. -To use `S3` locations, compile with the `s3` feature: +To use `S3Storage`, compile with the `s3-storage` feature: ```sh -cargo run -p htsget-axum --features s3 +cargo run -p htsget-axum --features s3-storage ``` -This will start a ticket server with `S3` locations and a bucket called `"data"`. +This will start a ticket server with `S3Storage` using a bucket called `"data"`. -To use `Url` locations, compile with the `url` feature. +To use `UrlStorage`, compile with the `url-storage` feature. See [htsget-search] for details on how to structure files. @@ -178,8 +178,8 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc #### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index b412bbbd..c12cf737 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-config/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3 = ["dep:aws-sdk-secretsmanager", "dep:aws-config", "dep:tempfile"] -url = ["dep:reqwest", "dep:cfg-if"] +s3-storage = ["dep:aws-sdk-secretsmanager", "dep:aws-config", "dep:tempfile"] +url-storage = ["dep:reqwest", "dep:cfg-if"] experimental = ["dep:crypt4gh", "dep:tokio", "dep:futures-util"] default = [] diff --git a/htsget-config/README.md b/htsget-config/README.md index 829ff723..560448f5 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -82,7 +82,7 @@ data_server = "None" > [!IMPORTANT] > Some parts of htsget-rs require extra feature flags for conditional compilation, that's why the examples specify -> using `--all-features`. Notably, `--features s3` enables the `S3` location type, and `--features url` +> using `--all-features`. Notably, `--features s3-storage` enables the `S3` location type, and `--features url-storage` > enabled the remote HTTP server location type. If using a subset of features, for example S3 locations only, then > a single feature can be enabled instead of using `--all-features`. @@ -384,7 +384,7 @@ location.keys.private = "data/c4gh/keys/bob.sec" # pragma: allowlist secret location.keys.public = "data/c4gh/keys/alice.pub" ``` -Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3` feature flag and specify `keys.kind = "SecretsManager"` under +Keys can also be retrieved from [AWS Secrets Manager][secrets-manager]. Compile with the `s3-storage` feature flag and specify `keys.kind = "SecretsManager"` under `location` to fetch keys from Secrets Manager. When using Secrets Manager, the `private` and `public` correspond to ARNs or secret names in Secrets Manager storing PEM formatted keys. @@ -442,8 +442,8 @@ regex, and changing it by using a substitution string. Advanced configuration op ### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 58c1dd74..7948e457 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -1,5 +1,5 @@ # An example for a server which uses s3 storage with data located in "bucket". -# Run with `cargo run -p htsget-axum --features s3 -- --config htsget-config/examples/config-files/s3_storage.toml` +# Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/s3_storage.toml` ticket_server.cors.allow_headers = "All" ticket_server.cors.allow_methods = "All" diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index fa99cd8a..d73bb596 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -1,5 +1,5 @@ # An example config file for a TLS ticket server that uses S3 as a storage backend. -# Run with `cargo run -p htsget-axum --features s3 -- --config htsget-config/examples/config-files/tls_ticket_server.toml` +# Run with `cargo run -p htsget-axum --features s3-storage -- --config htsget-config/examples/config-files/tls_ticket_server.toml` ticket_server.addr = "0.0.0.0:8080" ticket_server.cors_allow_origins = "All" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index 549a2ea0..078a41b4 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -1,6 +1,6 @@ # An example for a server which uses url storage with data located at "http://localhost:3000". # Run with -# `cargo run -p htsget-axum --features url -- --config htsget-config/examples/config-files/url_storage.toml` +# `cargo run -p htsget-axum --features url-storage -- --config htsget-config/examples/config-files/url_storage.toml` # in the project directory. ticket_server.addr = "127.0.0.1:8082" diff --git a/htsget-config/src/config/advanced/allow_guard.rs b/htsget-config/src/config/advanced/allow_guard.rs index 1fda3bab..3394ce93 100644 --- a/htsget-config/src/config/advanced/allow_guard.rs +++ b/htsget-config/src/config/advanced/allow_guard.rs @@ -150,7 +150,7 @@ impl QueryAllowed for AllowGuard { mod tests { use super::*; use crate::config::tests::test_serialize_and_deserialize; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use crate::config::Config; use crate::types::Class::Header; @@ -265,7 +265,7 @@ mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn allow_guard() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/advanced/mod.rs b/htsget-config/src/config/advanced/mod.rs index 6637dc86..a23342de 100644 --- a/htsget-config/src/config/advanced/mod.rs +++ b/htsget-config/src/config/advanced/mod.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; pub mod allow_guard; pub mod cors; pub mod regex_location; -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] pub mod url; /// Determines which tracing formatting style to use. diff --git a/htsget-config/src/config/advanced/regex_location.rs b/htsget-config/src/config/advanced/regex_location.rs index 1d6a72fc..d8bff741 100644 --- a/htsget-config/src/config/advanced/regex_location.rs +++ b/htsget-config/src/config/advanced/regex_location.rs @@ -98,7 +98,7 @@ mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn regex_location_s3() { test_serialize_and_deserialize( @@ -127,7 +127,7 @@ mod tests { ); } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[test] fn regex_location_url() { test_serialize_and_deserialize( @@ -158,7 +158,7 @@ mod tests { ); } - #[cfg(all(feature = "url", feature = "s3"))] + #[cfg(all(feature = "url-storage", feature = "s3-storage"))] #[test] fn regex_location_multiple() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/location.rs b/htsget-config/src/config/location.rs index 6e870a1e..7ca847f4 100644 --- a/htsget-config/src/config/location.rs +++ b/htsget-config/src/config/location.rs @@ -10,7 +10,7 @@ use crate::types::Scheme; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; use std::result; -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] use {crate::config::advanced::url::Url, crate::error, http::Uri}; /// The locations of data. @@ -214,7 +214,7 @@ impl<'de> Deserialize<'de> for StringLocation { }); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] if let Some(s) = s.strip_prefix("s3://") { let (bucket, prefix) = split(s)?; return Ok(StringLocation { @@ -223,7 +223,7 @@ impl<'de> Deserialize<'de> for StringLocation { }); } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] if let Some(s_stripped) = s .strip_prefix("http://") .or_else(|| s.strip_prefix("https://")) @@ -330,7 +330,7 @@ mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn location_s3() { test_serialize_and_deserialize( @@ -364,7 +364,7 @@ mod tests { ); } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[test] fn location_url() { test_serialize_and_deserialize( diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index 1e78e5f6..087bb4ca 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -221,7 +221,7 @@ pub(crate) mod tests { use crate::types::Scheme; use figment::Jail; use http::uri::Authority; - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] use http::Uri; use serde::de::DeserializeOwned; use serde_json::json; @@ -572,7 +572,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn simple_locations_s3() { test_config_from_file( @@ -590,7 +590,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[test] fn simple_locations_url() { test_config_from_file( @@ -621,7 +621,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn simple_locations_multiple_mixed() { test_config_from_file( diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index c1b1f5f4..b7183163 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -23,11 +23,11 @@ pub trait ResolveResponse { async fn from_file(file_storage: &storage::file::File, query: &Query) -> Result; /// Convert from `S3`. - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result; /// Convert from `Url`. - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] async fn from_url(url_storage: &storage::url::Url, query: &Query) -> Result; } @@ -122,7 +122,7 @@ impl StorageResolver for LocationEither { match self.backend() { Backend::File(file) => Some(T::from_file(file, query).await), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] Backend::S3(s3) => { let s3 = if let Self::Regex(regex_location) = self { if s3.bucket().is_empty() { @@ -142,7 +142,7 @@ impl StorageResolver for LocationEither { Some(T::from_s3(s3, query).await) } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] Backend::Url(url_storage) => Some(T::from_url(url_storage, query).await), } } @@ -200,9 +200,9 @@ mod tests { use crate::types::Scheme::Http; use crate::types::Url; use http::uri::Authority; - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] use reqwest::ClientBuilder; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use { crate::config::advanced::allow_guard::{AllowGuard, ReferenceNames}, crate::types::{Class, Fields, Interval, Tags}, @@ -220,7 +220,7 @@ mod tests { )) } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { Ok(Response::new( Bam, @@ -228,7 +228,7 @@ mod tests { )) } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] async fn from_url(url: &storage::url::Url, query: &Query) -> Result { Ok(Response::new( Bam, @@ -263,7 +263,7 @@ mod tests { expected_resolved_request(vec![location.into()], "127.0.0.1:8080/id-1").await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { let s3_storage = storage::s3::S3::new("id2".to_string(), None, false); @@ -279,7 +279,7 @@ mod tests { expected_resolved_request(vec![location.into()], "id2/id-1").await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request() { let regex_location = RegexLocation::new( @@ -305,7 +305,7 @@ mod tests { expected_resolved_request(vec![location.into()], "bucket/id-1").await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn resolver_resolve_url_request() { let client = ClientBuilder::new().build().unwrap(); @@ -440,7 +440,7 @@ mod tests { }); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn config_resolvers_all_options_env() { test_config_from_env( diff --git a/htsget-config/src/storage/c4gh/local.rs b/htsget-config/src/storage/c4gh/local.rs index 116343e6..696abc8b 100644 --- a/htsget-config/src/storage/c4gh/local.rs +++ b/htsget-config/src/storage/c4gh/local.rs @@ -98,7 +98,7 @@ mod tests { }); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn config_s3_storage_c4gh() { test_c4gh_storage_config( @@ -115,7 +115,7 @@ mod tests { ); } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn config_url_storage_c4gh() { test_c4gh_storage_config( diff --git a/htsget-config/src/storage/c4gh/mod.rs b/htsget-config/src/storage/c4gh/mod.rs index 9f5ab17d..1ef59329 100644 --- a/htsget-config/src/storage/c4gh/mod.rs +++ b/htsget-config/src/storage/c4gh/mod.rs @@ -4,7 +4,7 @@ use crate::error::Error::{IoError, ParseError}; use crate::error::{Error, Result}; use crate::storage::c4gh::local::C4GHLocal; -#[cfg(feature = "s3")] +#[cfg(feature = "s3-storage")] use crate::storage::c4gh::secrets_manager::C4GHSecretsManager; use crypt4gh::error::Crypt4GHError; use futures_util::future::{BoxFuture, Shared}; @@ -14,7 +14,7 @@ use tokio::task::{JoinError, JoinHandle}; pub mod local; -#[cfg(feature = "s3")] +#[cfg(feature = "s3-storage")] pub mod secrets_manager; /// Config for Crypt4GH keys. @@ -66,7 +66,7 @@ impl TryFrom for C4GHKeys { fn try_from(location: C4GHKeyLocation) -> Result { match location { C4GHKeyLocation::File(file) => file.try_into(), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] C4GHKeyLocation::SecretsManager(secrets_manager) => secrets_manager.try_into(), } } @@ -79,7 +79,7 @@ impl TryFrom for C4GHKeys { pub enum C4GHKeyLocation { #[serde(alias = "file", alias = "FILE")] File(C4GHLocal), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[serde(alias = "secretsmanager", alias = "SECRETSMANAGER")] SecretsManager(C4GHSecretsManager), } diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index 25c3f521..b8dba455 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,22 +1,22 @@ //! Storage backends. //! -#[cfg(any(feature = "url", feature = "s3"))] +#[cfg(any(feature = "url-storage", feature = "s3-storage"))] use crate::error::Error; use crate::error::Result; use crate::storage::file::File; -#[cfg(feature = "s3")] +#[cfg(feature = "s3-storage")] use crate::storage::s3::S3; -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] use crate::storage::url::Url; use serde::{Deserialize, Serialize}; #[cfg(feature = "experimental")] pub mod c4gh; pub mod file; -#[cfg(feature = "s3")] +#[cfg(feature = "s3-storage")] pub mod s3; -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] pub mod url; /// A new type representing a resolved id. @@ -42,10 +42,10 @@ impl ResolvedId { pub enum Backend { #[serde(alias = "file", alias = "FILE")] File(File), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] S3(S3), - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[serde(alias = "url", alias = "URL")] Url(Url), } @@ -55,15 +55,15 @@ impl Backend { pub fn as_file(&self) -> Result<&File> { match self { Backend::File(file) => Ok(file), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] Backend::S3(_) => Err(Error::ParseError("not a `File` variant".to_string())), - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] Backend::Url(_) => Err(Error::ParseError("not a `File` variant".to_string())), } } /// Get the file variant and error if it is not `S3`. - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] pub fn as_s3(&self) -> Result<&S3> { if let Backend::S3(s3) = self { Ok(s3) @@ -73,7 +73,7 @@ impl Backend { } /// Get the url variant and error if it is not `Url`. - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] pub fn as_url(&self) -> Result<&Url> { if let Backend::Url(url) = self { Ok(url) @@ -124,7 +124,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn config_storage_tagged_s3_file() { test_config_from_file( @@ -142,7 +142,7 @@ pub(crate) mod tests { ); } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[test] fn config_storage_tagged_s3_env() { test_config_from_env( diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs index 7518ea48..c1094cce 100644 --- a/htsget-config/src/tls/mod.rs +++ b/htsget-config/src/tls/mod.rs @@ -1,7 +1,7 @@ //! Configuration related to TLS. //! -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] pub mod client; use std::fs::File; diff --git a/htsget-http/Cargo.toml b/htsget-http/Cargo.toml index a2df23c9..6603f12f 100644 --- a/htsget-http/Cargo.toml +++ b/htsget-http/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-http/README.md" repository = "https://github.com/umccr/htsget-rs" [features] -s3 = ["htsget-config/s3", "htsget-search/s3", "htsget-test/s3"] -url = ["htsget-config/url", "htsget-search/url", "htsget-test/url"] +s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-test/s3-storage"] +url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-test/url-storage"] experimental = ["htsget-config/experimental", "htsget-search/experimental", "htsget-test/experimental"] default = [] diff --git a/htsget-http/README.md b/htsget-http/README.md index 5a8cd66a..a3cc059d 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -35,8 +35,8 @@ and process it using [htsget-search] to return JSON HTTP responses. #### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp diff --git a/htsget-lambda/Cargo.toml b/htsget-lambda/Cargo.toml index ad09ddce..8f3b479a 100644 --- a/htsget-lambda/Cargo.toml +++ b/htsget-lambda/Cargo.toml @@ -11,8 +11,8 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-lambda/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3 = ["htsget-axum/s3", "htsget-config/s3", "htsget-search/s3", "htsget-http/s3", "htsget-test/s3"] -url = ["htsget-axum/url", "htsget-config/url", "htsget-search/url", "htsget-http/url", "htsget-test/url"] +s3-storage = ["htsget-axum/s3-storage", "htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-test/s3-storage"] +url-storage = ["htsget-axum/url-storage", "htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-test/url-storage"] experimental = [ "htsget-axum/experimental", "htsget-config/experimental", diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index 2c72092f..9388ef06 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -11,16 +11,16 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-search/README.md repository = "https://github.com/umccr/htsget-rs" [features] -s3 = [ - "htsget-storage/s3", - "htsget-config/s3", - "htsget-test/s3", +s3-storage = [ + "htsget-storage/s3-storage", + "htsget-config/s3-storage", + "htsget-test/s3-storage", "htsget-test/aws-mocks" ] -url = [ - "htsget-storage/url", - "htsget-config/url", - "htsget-test/url" +url-storage = [ + "htsget-storage/url-storage", + "htsget-config/url-storage", + "htsget-test/url-storage" ] experimental = [ "htsget-storage/experimental", diff --git a/htsget-search/README.md b/htsget-search/README.md index 683e5c18..27149f16 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -53,8 +53,8 @@ used to process requests. #### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index e0a78e5a..4d80587a 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -147,7 +147,7 @@ impl BamSearch { #[cfg(test)] pub(crate) mod tests { use super::*; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Body, Class::Header, Headers, HtsGetError::NotFound, Response, Url}; @@ -535,7 +535,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -553,7 +553,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -572,7 +572,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index 3a17412a..d4431839 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -113,7 +113,7 @@ mod tests { use std::future::Future; use super::*; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::search::SearchAll; @@ -328,7 +328,7 @@ mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -346,7 +346,7 @@ mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -365,7 +365,7 @@ mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index 6400f38b..86121085 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -277,7 +277,7 @@ mod tests { use htsget_test::http::concat::ConcatResponse; use super::*; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Header, Headers, HtsGetError::NotFound, Response, Url}; @@ -542,7 +542,7 @@ mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -560,7 +560,7 @@ mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -579,7 +579,7 @@ mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index 37d13c99..ca72580b 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -56,14 +56,14 @@ impl ResolveResponse for HtsGetFromStorage { searcher.search(query.clone()).await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] async fn from_s3(s3_storage: &storage::s3::S3, query: &Query) -> Result { let storage = Storage::from_s3(s3_storage).await; let searcher = HtsGetFromStorage::new(storage?); searcher.search(query.clone()).await } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] async fn from_url(url_storage_config: &storage::url::Url, query: &Query) -> Result { let storage = Storage::from_url(url_storage_config).await; let searcher = HtsGetFromStorage::new(storage?); @@ -90,7 +90,7 @@ pub(crate) mod tests { use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use { htsget_storage::s3::S3Storage, htsget_test::aws_mocks::with_s3_test_server, std::fs::create_dir, }; @@ -318,7 +318,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] pub(crate) async fn with_aws_storage_fn(test: F, path: &str, copy_files: &[&str]) where F: FnOnce(Storage) -> Fut, @@ -341,7 +341,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] pub(crate) async fn with_aws_s3_storage_fn(test: F, folder_name: String, base_path: &Path) where F: FnOnce(Storage) -> Fut, diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index 5ac5c8bd..d83209e1 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -124,7 +124,7 @@ pub(crate) mod tests { use std::future::Future; use super::*; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::search::SearchAll; @@ -339,7 +339,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( @@ -357,7 +357,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( @@ -376,7 +376,7 @@ pub(crate) mod tests { .await } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn search_non_existent_id_header_aws() { with_aws_storage_fn( diff --git a/htsget-storage/Cargo.toml b/htsget-storage/Cargo.toml index 89f92eed..a9d914bc 100644 --- a/htsget-storage/Cargo.toml +++ b/htsget-storage/Cargo.toml @@ -11,19 +11,19 @@ homepage = "https://github.com/umccr/htsget-rs/blob/main/htsget-storage/README.m repository = "https://github.com/umccr/htsget-rs" [features] -s3 = [ +s3-storage = [ "dep:bytes", "dep:aws-sdk-s3", "dep:aws-config", - "htsget-config/s3", - "htsget-test/s3", + "htsget-config/s3-storage", + "htsget-test/s3-storage", "htsget-test/aws-mocks" ] -url = [ +url-storage = [ "dep:bytes", "dep:reqwest", - "htsget-config/url", - "htsget-test/url" + "htsget-config/url-storage", + "htsget-test/url-storage" ] experimental = ["dep:crypt4gh", "dep:bincode", "htsget-config/experimental", "htsget-test/experimental"] default = [] diff --git a/htsget-storage/README.md b/htsget-storage/README.md index a606047b..5e035db7 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -46,8 +46,8 @@ and [url] modules implement the `Storage` functionality. #### Feature flags This crate has the following features: -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [local]: src/local.rs diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index 6063b276..d450425b 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -315,9 +315,9 @@ impl From for StorageError { mod tests { use super::*; use crate::local::tests::with_local_storage; - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] use crate::s3::tests::with_aws_s3_storage; - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] use crate::url::tests::{test_headers, with_url_test_server}; use htsget_config::types::Headers; use htsget_test::c4gh::{encrypt_data, get_decryption_keys}; @@ -335,7 +335,7 @@ mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn test_preprocess_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -344,7 +344,7 @@ mod tests { .await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn test_preprocess_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -363,7 +363,7 @@ mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn test_get_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -372,7 +372,7 @@ mod tests { .await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn test_get_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -391,7 +391,7 @@ mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn test_head_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -400,7 +400,7 @@ mod tests { .await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn test_head_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -419,7 +419,7 @@ mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn test_postprocess_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -428,7 +428,7 @@ mod tests { .await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn test_postprocess_url_storage() { with_url_c4gh_storage(|mut storage, _| async move { @@ -453,7 +453,7 @@ mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[tokio::test] async fn test_range_s3_storage() { with_s3_c4gh_storage(|mut storage| async move { @@ -468,7 +468,7 @@ mod tests { .await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] #[tokio::test] async fn test_range_url_storage() { with_url_c4gh_storage(|mut storage, url| async move { @@ -608,7 +608,7 @@ mod tests { .await; } - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] pub(crate) async fn with_s3_c4gh_storage(test: F) where F: FnOnce(C4GHStorage) -> Fut, @@ -621,7 +621,7 @@ mod tests { .await; } - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] pub(crate) async fn with_url_c4gh_storage(test: F) where F: FnOnce(C4GHStorage, String) -> Fut, diff --git a/htsget-storage/src/error.rs b/htsget-storage/src/error.rs index cce3e3a2..2f02d788 100644 --- a/htsget-storage/src/error.rs +++ b/htsget-storage/src/error.rs @@ -40,7 +40,7 @@ pub enum StorageError { #[error("response error: {0}")] ResponseError(String), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] #[error("aws error: {0}, with key: `{1}`")] AwsS3Error(String, String), @@ -60,7 +60,7 @@ impl From for HtsGetError { | StorageError::InvalidUri(_) | StorageError::InvalidAddress(_) | StorageError::InternalError(_)) => Self::InternalError(err.to_string()), - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] err @ StorageError::AwsS3Error(_, _) => Self::IoError(err.to_string()), err @ StorageError::UrlParseError(_) => Self::ParseError(err.to_string()), } diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index eead16e0..bb985df8 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -12,10 +12,10 @@ use crate::error::Result; use crate::error::StorageError; use crate::error::StorageError::InvalidKey; use crate::local::FileStorage; -#[cfg(feature = "s3")] +#[cfg(feature = "s3-storage")] use crate::s3::S3Storage; use crate::types::{BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions}; -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] use crate::url::UrlStorage; use async_trait::async_trait; use base64::engine::general_purpose; @@ -38,10 +38,10 @@ use tokio::io::{AsyncRead, ReadBuf}; pub mod c4gh; pub mod error; pub mod local; -#[cfg(feature = "s3")] +#[cfg(feature = "s3-storage")] pub mod s3; pub mod types; -#[cfg(feature = "url")] +#[cfg(feature = "url-storage")] pub mod url; pin_project! { @@ -163,7 +163,7 @@ impl Storage { } /// Create from s3 config. - #[cfg(feature = "s3")] + #[cfg(feature = "s3-storage")] pub async fn from_s3(s3: &storage::s3::S3) -> Result { let storage = Storage::new( S3Storage::new_with_default_config( @@ -184,7 +184,7 @@ impl Storage { } /// Create from url config. - #[cfg(feature = "url")] + #[cfg(feature = "url-storage")] pub async fn from_url(url: &storage::url::Url) -> Result { let storage = Storage::new(UrlStorage::new( url.client_cloned(), diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index c22663da..8f3dffe7 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -25,7 +25,7 @@ http = [ "dep:base64" ] aws-mocks = [ - "s3", + "s3-storage", "dep:tempfile", "dep:aws-sdk-s3", "dep:aws-config", @@ -34,8 +34,8 @@ aws-mocks = [ "dep:s3s-fs", "dep:s3s-aws" ] -s3 = ["htsget-config?/s3"] -url = ["htsget-config?/url"] +s3-storage = ["htsget-config?/s3-storage"] +url-storage = ["htsget-config?/url-storage"] experimental = ["dep:crypt4gh", "dep:htsget-config", "htsget-config/experimental"] default = [] diff --git a/htsget-test/README.md b/htsget-test/README.md index 14441a20..b3503ae8 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -34,8 +34,8 @@ This library is intended to be used as a [development dependency][dev-dependenci This crate has the following features: * `http`: used to enable common functionality for HTTP tests. * `aws-mocks`: used to enable AWS mocking for tests. -* `s3`: used to enable `S3` location functionality. -* `url`: used to enable `Url` location functionality. +* `s3-storage`: used to enable `S3` location functionality. +* `url-storage`: used to enable `Url` location functionality. * `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies