diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 542677de1..3efc08bd8 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -27,6 +27,9 @@ worker_processes: 8 # Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] cache_size_mb: 1024 +# If the client accepts multiple compression formats, and the tile source is not pre-compressed, which compression should be used. `gzip` is faster, but `brotli` is smaller, and may be faster with caching. Defaults to brotli. +preferred_encoding: gzip + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string. You can use env vars too, for example: diff --git a/docs/src/run-with-cli.md b/docs/src/run-with-cli.md index fd921c138..b52935580 100644 --- a/docs/src/run-with-cli.md +++ b/docs/src/run-with-cli.md @@ -31,6 +31,11 @@ Options: -W, --workers Number of web server workers + --preferred-encoding + Martin server preferred tile encoding. If the client accepts multiple compression formats, and the tile source is not pre-compressed, which compression should be used. `gzip` is faster, but `brotli` is smaller, and may be faster with caching. Defaults to brotli + + [possible values: brotli, gzip] + -b, --auto-bounds Specify how bounds should be computed for the spatial PG tables. [DEFAULT: quick] diff --git a/martin/benches/bench.rs b/martin/benches/bench.rs index 27f4904b2..ceb4a6946 100644 --- a/martin/benches/bench.rs +++ b/martin/benches/bench.rs @@ -58,7 +58,7 @@ impl Source for NullSource { } async fn process_tile(sources: &TileSources) { - let src = DynTileSource::new(sources, "null", Some(0), "", None, None).unwrap(); + let src = DynTileSource::new(sources, "null", Some(0), "", None, None, None).unwrap(); src.get_http_response(TileCoord { z: 0, x: 0, y: 0 }) .await .unwrap(); diff --git a/martin/src/args/mod.rs b/martin/src/args/mod.rs index f3bb5e81e..0e58bd7ab 100644 --- a/martin/src/args/mod.rs +++ b/martin/src/args/mod.rs @@ -13,4 +13,5 @@ mod root; pub use root::{Args, ExtraArgs, MetaArgs}; mod srv; +pub use srv::PreferredEncoding; pub use srv::SrvArgs; diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 7fc81d1be..7bcda8103 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -160,7 +160,9 @@ pub fn parse_file_args( #[cfg(test)] mod tests { + use super::*; + use crate::args::PreferredEncoding; use crate::test_utils::FauxEnv; use crate::MartinError::UnrecognizableConnections; @@ -215,6 +217,28 @@ mod tests { assert_eq!(args, (cfg, meta)); } + #[test] + fn cli_encoding_arguments() { + let config1 = parse(&["martin", "--preferred-encoding", "brotli"]); + let config2 = parse(&["martin", "--preferred-encoding", "br"]); + let config3 = parse(&["martin", "--preferred-encoding", "gzip"]); + let config4 = parse(&["martin"]); + + assert_eq!( + config1.unwrap().0.srv.preferred_encoding, + Some(PreferredEncoding::Brotli) + ); + assert_eq!( + config2.unwrap().0.srv.preferred_encoding, + Some(PreferredEncoding::Brotli) + ); + assert_eq!( + config3.unwrap().0.srv.preferred_encoding, + Some(PreferredEncoding::Gzip) + ); + assert_eq!(config4.unwrap().0.srv.preferred_encoding, None); + } + #[test] fn cli_bad_arguments() { for params in [ diff --git a/martin/src/args/srv.rs b/martin/src/args/srv.rs index b436151b3..e7b743583 100644 --- a/martin/src/args/srv.rs +++ b/martin/src/args/srv.rs @@ -1,4 +1,6 @@ use crate::srv::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT}; +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; #[derive(clap::Args, Debug, PartialEq, Default)] #[command(about, version)] @@ -10,6 +12,19 @@ pub struct SrvArgs { /// Number of web server workers #[arg(short = 'W', long)] pub workers: Option, + /// Martin server preferred tile encoding. If the client accepts multiple compression formats, and the tile source is not pre-compressed, which compression should be used. `gzip` is faster, but `brotli` is smaller, and may be faster with caching. Defaults to brotli. + #[arg(long)] + pub preferred_encoding: Option, +} + +#[derive(PartialEq, Eq, Default, Debug, Clone, Copy, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum PreferredEncoding { + #[default] + #[serde(alias = "br")] + #[clap(alias("br"))] + Brotli, + Gzip, } impl SrvArgs { @@ -24,5 +39,8 @@ impl SrvArgs { if self.workers.is_some() { srv_config.worker_processes = self.workers; } + if self.preferred_encoding.is_some() { + srv_config.preferred_encoding = self.preferred_encoding; + } } } diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 35c75d6ce..ecb3a291f 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -283,6 +283,7 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> args.url_query.as_deref().unwrap_or_default(), Some(parse_encoding(args.encoding.as_str())?), None, + None, )?; // parallel async below uses move, so we must only use copyable types let src = &src; diff --git a/martin/src/srv/config.rs b/martin/src/srv/config.rs index 0db8db587..bef288160 100644 --- a/martin/src/srv/config.rs +++ b/martin/src/srv/config.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +use crate::args::PreferredEncoding; + pub const KEEP_ALIVE_DEFAULT: u64 = 75; pub const LISTEN_ADDRESSES_DEFAULT: &str = "0.0.0.0:3000"; @@ -9,6 +11,7 @@ pub struct SrvConfig { pub keep_alive: Option, pub listen_addresses: Option, pub worker_processes: Option, + pub preferred_encoding: Option, } #[cfg(test)] @@ -19,18 +22,49 @@ mod tests { use crate::test_utils::some; #[test] - fn parse_empty_config() { + fn parse_config() { + assert_eq!( + serde_yaml::from_str::(indoc! {" + keep_alive: 75 + listen_addresses: '0.0.0.0:3000' + worker_processes: 8 + "}) + .unwrap(), + SrvConfig { + keep_alive: Some(75), + listen_addresses: some("0.0.0.0:3000"), + worker_processes: Some(8), + preferred_encoding: None, + } + ); + assert_eq!( + serde_yaml::from_str::(indoc! {" + keep_alive: 75 + listen_addresses: '0.0.0.0:3000' + worker_processes: 8 + preferred_encoding: br + "}) + .unwrap(), + SrvConfig { + keep_alive: Some(75), + listen_addresses: some("0.0.0.0:3000"), + worker_processes: Some(8), + preferred_encoding: Some(PreferredEncoding::Brotli), + } + ); assert_eq!( serde_yaml::from_str::(indoc! {" keep_alive: 75 listen_addresses: '0.0.0.0:3000' worker_processes: 8 + preferred_encoding: brotli "}) .unwrap(), SrvConfig { keep_alive: Some(75), listen_addresses: some("0.0.0.0:3000"), worker_processes: Some(8), + preferred_encoding: Some(PreferredEncoding::Brotli), } ); } diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index b0c1caa16..c187662e7 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -108,6 +108,13 @@ type Server = Pin>>>; pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server, String)> { let catalog = Catalog::new(&state)?; + let keep_alive = Duration::from_secs(config.keep_alive.unwrap_or(KEEP_ALIVE_DEFAULT)); + let worker_processes = config.worker_processes.unwrap_or_else(num_cpus::get); + let listen_addresses = config + .listen_addresses + .clone() + .unwrap_or_else(|| LISTEN_ADDRESSES_DEFAULT.to_owned()); + let factory = move || { let cors_middleware = Cors::default() .allow_any_origin() @@ -124,6 +131,7 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server let app = app.app_data(Data::new(state.fonts.clone())); app.app_data(Data::new(catalog.clone())) + .app_data(Data::new(config.clone())) .wrap(cors_middleware) .wrap(middleware::NormalizePath::new(TrailingSlash::MergeOnly)) .wrap(middleware::Logger::default()) @@ -136,12 +144,6 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server return Ok((Box::pin(server), "(aws lambda)".into())); } - let keep_alive = Duration::from_secs(config.keep_alive.unwrap_or(KEEP_ALIVE_DEFAULT)); - let worker_processes = config.worker_processes.unwrap_or_else(num_cpus::get); - let listen_addresses = config - .listen_addresses - .unwrap_or_else(|| LISTEN_ADDRESSES_DEFAULT.to_owned()); - let server = HttpServer::new(factory) .bind(listen_addresses.clone()) .map_err(|e| BindingError(e, listen_addresses.clone()))? diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs index f25829966..c5195b87c 100755 --- a/martin/src/srv/tiles.rs +++ b/martin/src/srv/tiles.rs @@ -10,8 +10,10 @@ use log::trace; use martin_tile_utils::{Encoding, Format, TileInfo}; use serde::Deserialize; +use crate::args::PreferredEncoding; use crate::source::{Source, TileSources, UrlQuery}; use crate::srv::server::map_internal_error; +use crate::srv::SrvConfig; use crate::utils::cache::get_or_insert_cached_value; use crate::utils::{ decode_brotli, decode_gzip, encode_brotli, encode_gzip, CacheKey, CacheValue, MainCache, @@ -19,12 +21,18 @@ use crate::utils::{ }; use crate::{Tile, TileCoord, TileData}; -static SUPPORTED_ENCODINGS: &[HeaderEnc] = &[ +static PREFER_BROTLI_ENC: &[HeaderEnc] = &[ HeaderEnc::brotli(), HeaderEnc::gzip(), HeaderEnc::identity(), ]; +static PREFER_GZIP_ENC: &[HeaderEnc] = &[ + HeaderEnc::gzip(), + HeaderEnc::brotli(), + HeaderEnc::identity(), +]; + #[derive(Deserialize, Clone)] pub struct TileRequest { source_ids: String, @@ -36,6 +44,7 @@ pub struct TileRequest { #[route("/{source_ids}/{z}/{x}/{y}", method = "GET", method = "HEAD")] async fn get_tile( req: HttpRequest, + srv_config: Data, path: Path, sources: Data, cache: Data, @@ -46,6 +55,7 @@ async fn get_tile( Some(path.z), req.query_string(), req.get_header::(), + srv_config.preferred_encoding, cache.as_ref().as_ref(), )?; @@ -62,7 +72,8 @@ pub struct DynTileSource<'a> { pub info: TileInfo, pub query_str: Option<&'a str>, pub query_obj: Option, - pub encodings: Option, + pub accept_enc: Option, + pub preferred_enc: Option, pub cache: Option<&'a MainCache>, } @@ -72,7 +83,8 @@ impl<'a> DynTileSource<'a> { source_ids: &str, zoom: Option, query: &'a str, - encodings: Option, + accept_enc: Option, + preferred_enc: Option, cache: Option<&'a MainCache>, ) -> ActixResult { let (sources, use_url_query, info) = sources.get_sources(source_ids, zoom)?; @@ -93,7 +105,8 @@ impl<'a> DynTileSource<'a> { info, query_str, query_obj, - encodings, + accept_enc, + preferred_enc, cache, }) } @@ -169,7 +182,7 @@ impl<'a> DynTileSource<'a> { fn recompress(&self, tile: TileData) -> ActixResult { let mut tile = Tile::new(tile, self.info); - if let Some(accept_enc) = &self.encodings { + if let Some(accept_enc) = &self.accept_enc { if self.info.encoding.is_encoded() { // already compressed, see if we can send it as is, or need to re-compress if !accept_enc.iter().any(|e| { @@ -183,10 +196,15 @@ impl<'a> DynTileSource<'a> { tile = decode(tile)?; } } + if tile.info.encoding == Encoding::Uncompressed { + let ordered_encodings = match self.preferred_enc { + Some(PreferredEncoding::Gzip) => PREFER_GZIP_ENC, + Some(PreferredEncoding::Brotli) | None => PREFER_BROTLI_ENC, + }; + // only apply compression if the content supports it - if let Some(HeaderEnc::Known(enc)) = - accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) + if let Some(HeaderEnc::Known(enc)) = accept_enc.negotiate(ordered_encodings.iter()) { // (re-)compress the tile into the preferred encoding tile = encode(tile, enc)?; @@ -251,6 +269,53 @@ mod tests { use super::*; use crate::srv::server::tests::TestSource; + #[actix_rt::test] + async fn test_encoding_preference() { + let source = TestSource { + id: "test_source", + tj: tilejson! { tiles: vec![] }, + data: vec![1_u8, 2, 3], + }; + let sources = TileSources::new(vec![vec![Box::new(source)]]); + + for (accept_encodings, prefered_encoding, result_encoding) in [ + ( + Some(AcceptEncoding(vec![ + "gzip;q=1".parse().unwrap(), + "br;q=1".parse().unwrap(), + ])), + Some(PreferredEncoding::Brotli), + Encoding::Brotli, + ), + ( + Some(AcceptEncoding(vec![ + "gzip;q=1".parse().unwrap(), + "br;q=0.5".parse().unwrap(), + ])), + Some(PreferredEncoding::Brotli), + Encoding::Gzip, + ), + ] { + let src = DynTileSource::new( + &sources, + "test_source", + None, + "", + accept_encodings, + prefered_encoding, + None, + ) + .unwrap(); + let xyz = TileCoord { z: 0, x: 0, y: 0 }; + let data = &src.get_tile_content(xyz).await.unwrap().data; + let decoded = match result_encoding { + Encoding::Gzip => decode_gzip(data), + Encoding::Brotli => decode_brotli(data), + _ => panic!("Unexpected encoding"), + }; + assert_eq!(vec![1_u8, 2, 3], decoded.unwrap()); + } + } #[actix_rt::test] async fn test_tile_content() { let non_empty_source = TestSource { @@ -278,7 +343,7 @@ mod tests { ("empty,non-empty", vec![1_u8, 2, 3]), ("empty,non-empty,empty", vec![1_u8, 2, 3]), ] { - let src = DynTileSource::new(&sources, source_id, None, "", None, None).unwrap(); + let src = DynTileSource::new(&sources, source_id, None, "", None, None, None).unwrap(); let xyz = TileCoord { z: 0, x: 0, y: 0 }; assert_eq!(expected, &src.get_tile_content(xyz).await.unwrap().data); } diff --git a/martin/tests/mb_server_test.rs b/martin/tests/mb_server_test.rs index b38d490b6..9af72bc9a 100644 --- a/martin/tests/mb_server_test.rs +++ b/martin/tests/mb_server_test.rs @@ -4,6 +4,7 @@ use ctor::ctor; use indoc::indoc; use insta::assert_yaml_snapshot; use martin::decode_gzip; +use martin::srv::SrvConfig; use tilejson::TileJSON; pub mod utils; @@ -24,6 +25,7 @@ macro_rules! create_app { )) .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) + .app_data(actix_web::web::Data::new(SrvConfig::default())) .configure(::martin::srv::router), ) .await diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index 55b7f37a3..c63c11a7d 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -6,6 +6,7 @@ use actix_web::test::{call_and_read_body_json, call_service, read_body, TestRequ use ctor::ctor; use indoc::indoc; use insta::assert_yaml_snapshot; +use martin::srv::SrvConfig; use martin::OptOneMany; use tilejson::TileJSON; @@ -28,6 +29,7 @@ macro_rules! create_app { )) .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) + .app_data(actix_web::web::Data::new(SrvConfig::default())) .configure(::martin::srv::router), ) .await @@ -1089,6 +1091,7 @@ tables: )) .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) + .app_data(actix_web::web::Data::new(SrvConfig::default())) .configure(::martin::srv::router), ) .await; diff --git a/martin/tests/pmt_server_test.rs b/martin/tests/pmt_server_test.rs index 9a6e7ce46..422d82cde 100644 --- a/martin/tests/pmt_server_test.rs +++ b/martin/tests/pmt_server_test.rs @@ -4,6 +4,7 @@ use ctor::ctor; use indoc::indoc; use insta::assert_yaml_snapshot; use martin::decode_gzip; +use martin::srv::SrvConfig; use tilejson::TileJSON; pub mod utils; @@ -24,6 +25,7 @@ macro_rules! create_app { )) .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) + .app_data(actix_web::web::Data::new(SrvConfig::default())) .configure(::martin::srv::router), ) .await