From 89ef8083536cece02b43810a2a29becdeefe9e06 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sun, 24 Dec 2023 23:11:22 -0500 Subject: [PATCH] PMTiles cache, refactor file configs, modularize * Implement PMTiles directory cache shared between all pmtiles, with configurable max cache size (in MB), or 0 to disable. * PMTiles now share web client instance, which optimizes connection reuse in case multiple pmtiles reside on the same host * Major refactoring to allow modular reuse, enabling the following build features: * **postgres** - enable PostgreSQL/PostGIS tile sources * **pmtiles** - enable PMTile tile sources * **mbtiles** - enable MBTile tile sources * **fonts** - enable font sources * **sprites** - enable sprite sources * Use justfile in the CI --- Cargo.lock | 40 ++-- Cargo.toml | 4 +- README.md | 8 + debian/config.yaml | 5 +- docs/src/config-file.md | 2 + justfile | 11 +- martin-tile-utils/Cargo.toml | 2 +- martin/Cargo.toml | 34 +-- martin/src/args/mod.rs | 2 + martin/src/args/root.rs | 46 ++-- martin/src/bin/martin-cp.rs | 6 +- martin/src/config.rs | 50 ++-- martin/src/file_config.rs | 185 +++++---------- martin/src/lib.rs | 3 + martin/src/mbtiles/mod.rs | 104 ++++++-- martin/src/pg/config.rs | 3 +- martin/src/pg/configurator.rs | 2 +- martin/src/pg/utils.rs | 35 +++ martin/src/pmtiles/file_pmtiles.rs | 58 +++++ martin/src/pmtiles/http_pmtiles.rs | 42 ++++ martin/src/pmtiles/mod.rs | 264 +++++++++++---------- martin/src/sprites/mod.rs | 23 +- martin/src/utils/error.rs | 4 +- martin/src/utils/utilities.rs | 35 --- martin/tests/pg_function_source_test.rs | 2 + martin/tests/pg_server_test.rs | 2 + martin/tests/pg_table_source_test.rs | 2 + martin/tests/utils/pg_utils.rs | 6 +- mbtiles/Cargo.toml | 2 +- tests/config.yaml | 2 +- tests/expected/configured/save_config.yaml | 2 +- 31 files changed, 585 insertions(+), 401 deletions(-) create mode 100644 martin/src/pmtiles/file_pmtiles.rs create mode 100644 martin/src/pmtiles/http_pmtiles.rs diff --git a/Cargo.lock b/Cargo.lock index 44c910c01..1b2b187d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,9 +85,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http", @@ -845,9 +845,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" +checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -866,21 +866,20 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -888,9 +887,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if", ] @@ -1963,7 +1962,7 @@ dependencies = [ [[package]] name = "martin" -version = "0.11.6" +version = "0.12.0" dependencies = [ "actix-cors", "actix-http", @@ -2015,7 +2014,7 @@ dependencies = [ [[package]] name = "martin-tile-utils" -version = "0.3.1" +version = "0.4.0" dependencies = [ "approx", "insta", @@ -2023,7 +2022,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.8.5" +version = "0.9.0" dependencies = [ "actix-rt", "anyhow", @@ -2084,15 +2083,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -2275,9 +2265,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 0e7040231..a4b465058 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,8 +48,8 @@ insta = "1" itertools = "0.12" json-patch = "1.2" log = "0.4" -martin-tile-utils = { path = "./martin-tile-utils", version = "0.3.0" } -mbtiles = { path = "./mbtiles", version = "0.8.0" } +martin-tile-utils = { path = "./martin-tile-utils", version = "0.4.0" } +mbtiles = { path = "./mbtiles", version = "0.9.0" } moka = { version = "0.12", features = ["future"] } num_cpus = "1" pbf_font_tools = { version = "2.5.0", features = ["freetype"] } diff --git a/README.md b/README.md index 81fb3b42a..5e7f44688 100755 --- a/README.md +++ b/README.md @@ -106,6 +106,14 @@ Martin data is available via the HTTP `GET` endpoints: | `/font/{font1},…,{fontN}/{start}-{end}` | Composite Font source | | `/health` | Martin server health check: returns 200 `OK` | +## Re-use Martin as a library +Martin can be used as a standalone server, or as a library in your own Rust application. When used as a library, you can use the following features: +* **postgres** - enable PostgreSQL/PostGIS tile sources +* **pmtiles** - enable PMTile tile sources +* **mbtiles** - enable MBTile tile sources +* **fonts** - enable font sources +* **sprites** - enable sprite sources + ## Documentation See [Martin book](https://maplibre.org/martin/) for complete documentation. diff --git a/debian/config.yaml b/debian/config.yaml index a363978d6..b597a13bd 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -17,7 +17,7 @@ worker_processes: 8 # auto_bounds: skip # pmtiles: -# dir_cache_size: 100 +# dir_cache_size_mb: 100 # paths: # - /dir-path # - /path/to/pmtiles.pmtiles @@ -33,6 +33,9 @@ worker_processes: 8 # sources: # mb-src1: /path/to/mbtiles1.mbtiles +# sprites: +# - /path/to/sprites_dir + # fonts: # - /path/to/font/file.ttf # - /path/to/font_dir diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 008b1a950..19cfec94f 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -155,6 +155,8 @@ postgres: # Publish PMTiles files from local disk or proxy to a web server pmtiles: + # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] + dir_cache_size_mb: 100 paths: # scan this whole dir, matching all *.pmtiles files - /dir-path diff --git a/justfile b/justfile index 7a1af4369..64462ac1e 100644 --- a/justfile +++ b/justfile @@ -273,7 +273,16 @@ fmt2: # Run cargo check check: - cargo check --workspace --all-targets --bins --tests --lib --benches + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin-tile-utils + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p mbtiles + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p mbtiles --no-default-features + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features fonts + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features mbtiles + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features pmtiles + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features postgres + RUSTFLAGS='-D warnings' cargo check --bins --tests --lib --benches --examples -p martin --no-default-features --features sprites # Verify doc build check-doc: diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index d03c158b4..1f6899566 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "martin-tile-utils" -version = "0.3.1" +version = "0.4.0" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server." keywords = ["maps", "tiles", "mvt", "tileserver"] diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 3d58e3cae..0f73937f8 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -3,7 +3,7 @@ lints.workspace = true [package] name = "martin" # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin -version = "0.11.6" +version = "0.12.0" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"] @@ -59,10 +59,12 @@ name = "bench" harness = false [features] -default = [] -#default = ["sprites", "fonts"] -sprites = [] -fonts = [] +default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] +fonts = ["dep:bit-set","dep:pbf_font_tools"] +mbtiles = [] +pmtiles = ["dep:moka"] +postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] +sprites = ["dep:spreet"] bless-tests = [] [dependencies] @@ -71,41 +73,41 @@ actix-http.workspace = true actix-rt.workspace = true actix-web.workspace = true async-trait.workspace = true -bit-set.workspace = true +bit-set = { workspace = true, optional = true } brotli.workspace = true clap.workspace = true -deadpool-postgres.workspace = true +deadpool-postgres = { workspace = true, optional = true } env_logger.workspace = true flate2.workspace = true futures.workspace = true itertools.workspace = true -json-patch.workspace = true +json-patch = { workspace = true, optional = true } log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true -moka.workspace = true +moka = { workspace = true, optional = true } num_cpus.workspace = true -pbf_font_tools.workspace = true +pbf_font_tools = { workspace = true, optional = true } pmtiles.workspace = true -postgis.workspace = true -postgres-protocol.workspace = true -postgres.workspace = true +postgis = { workspace = true, optional = true } +postgres-protocol = { workspace = true, optional = true } +postgres = { workspace = true, optional = true } regex.workspace = true reqwest.workspace = true rustls-native-certs.workspace = true rustls-pemfile.workspace = true rustls.workspace = true -semver.workspace = true +semver = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true serde_with.workspace = true serde_yaml.workspace = true -spreet.workspace = true +spreet = { workspace = true, optional = true } subst.workspace = true thiserror.workspace = true tilejson.workspace = true tokio = { workspace = true, features = ["io-std"] } -tokio-postgres-rustls.workspace = true +tokio-postgres-rustls = { workspace = true, optional = true } url.workspace = true [dev-dependencies] diff --git a/martin/src/args/mod.rs b/martin/src/args/mod.rs index fb1b3c34c..f3bb5e81e 100644 --- a/martin/src/args/mod.rs +++ b/martin/src/args/mod.rs @@ -4,7 +4,9 @@ pub use connections::{Arguments, State}; mod environment; pub use environment::{Env, OsEnv}; +#[cfg(feature = "postgres")] mod pg; +#[cfg(feature = "postgres")] pub use pg::{BoundsCalcType, PgArgs, DEFAULT_BOUNDS_TIMEOUT}; mod root; diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index e4e3871ec..518964009 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -2,15 +2,13 @@ use std::path::PathBuf; use clap::Parser; use log::warn; -use url::Url; use crate::args::connections::Arguments; use crate::args::environment::Env; -use crate::args::pg::PgArgs; use crate::args::srv::SrvArgs; -use crate::args::State::{Ignore, Share, Take}; use crate::config::Config; -use crate::file_config::{FileConfigEnum, FileConfigExtras}; +#[cfg(any(feature = "mbtiles", feature = "pmtiles", feature = "sprites"))] +use crate::file_config::FileConfigEnum; use crate::MartinError::ConfigAndConnectionsError; use crate::{MartinResult, OptOneMany}; @@ -27,8 +25,9 @@ pub struct Args { pub extras: ExtraArgs, #[command(flatten)] pub srv: SrvArgs, + #[cfg(feature = "postgres")] #[command(flatten)] - pub pg: Option, + pub pg: Option, } // None of these params will be transferred to the config @@ -80,19 +79,26 @@ impl Args { self.srv.merge_into_config(&mut config.srv); + #[allow(unused_mut)] let mut cli_strings = Arguments::new(self.meta.connection); - let pg_args = self.pg.unwrap_or_default(); - if config.postgres.is_none() { - config.postgres = pg_args.into_config(&mut cli_strings, env); - } else { - // config was loaded from a file, we can only apply a few CLI overrides to it - pg_args.override_config(&mut config.postgres, env); + + #[cfg(feature = "postgres")] + { + let pg_args = self.pg.unwrap_or_default(); + if config.postgres.is_none() { + config.postgres = pg_args.into_config(&mut cli_strings, env); + } else { + // config was loaded from a file, we can only apply a few CLI overrides to it + pg_args.override_config(&mut config.postgres, env); + } } + #[cfg(feature = "pmtiles")] if !cli_strings.is_empty() { config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles", true); } + #[cfg(feature = "mbtiles")] if !cli_strings.is_empty() { config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles", false); } @@ -110,9 +116,10 @@ impl Args { } } +#[cfg(any(feature = "pmtiles", feature = "mbtiles"))] fn is_url(s: &str, extension: &str) -> bool { if s.starts_with("http") { - if let Ok(url) = Url::parse(s) { + if let Ok(url) = url::Url::parse(s) { if url.scheme() == "http" || url.scheme() == "https" { if let Some(ext) = url.path().rsplit('.').next() { return ext == extension; @@ -123,11 +130,14 @@ fn is_url(s: &str, extension: &str) -> bool { false } -pub fn parse_file_args( +#[cfg(any(feature = "pmtiles", feature = "mbtiles"))] +pub fn parse_file_args( cli_strings: &mut Arguments, extension: &str, allow_url: bool, ) -> FileConfigEnum { + use crate::args::State::{Ignore, Share, Take}; + let paths = cli_strings.process(|s| match PathBuf::try_from(s) { Ok(v) => { if allow_url && is_url(s, extension) { @@ -149,9 +159,7 @@ pub fn parse_file_args( #[cfg(test)] mod tests { use super::*; - use crate::pg::PgConfig; - use crate::test_utils::{some, FauxEnv}; - use crate::utils::OptOneMany; + use crate::test_utils::FauxEnv; use crate::MartinError::UnrecognizableConnections; fn parse(args: &[&str]) -> MartinResult<(Config, MetaArgs)> { @@ -169,8 +177,12 @@ mod tests { assert_eq!(args, expected); } + #[cfg(feature = "postgres")] #[test] fn cli_with_config() { + use crate::test_utils::some; + use crate::utils::OptOneMany; + let args = parse(&["martin", "--config", "c.toml"]).unwrap(); let meta = MetaArgs { config: Some(PathBuf::from("c.toml")), @@ -188,7 +200,7 @@ mod tests { let args = parse(&["martin", "postgres://connection"]).unwrap(); let cfg = Config { - postgres: OptOneMany::One(PgConfig { + postgres: OptOneMany::One(crate::pg::PgConfig { connection_string: some("postgres://connection"), ..Default::default() }), diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index cb4a88a72..2a2077a6a 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -11,7 +11,7 @@ use clap::Parser; use futures::stream::{self, StreamExt}; use futures::TryStreamExt; use log::{debug, error, info, log_enabled}; -use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, PgArgs, SrvArgs}; +use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs}; use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS}; use martin::{ append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source, @@ -46,8 +46,9 @@ pub struct CopierArgs { pub copy: CopyArgs, #[command(flatten)] pub meta: MetaArgs, + #[cfg(feature = "postgres")] #[command(flatten)] - pub pg: Option, + pub pg: Option, } #[serde_with::serde_as] @@ -137,6 +138,7 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { meta: copy_args.meta, extras: ExtraArgs::default(), srv: SrvArgs::default(), + #[cfg(feature = "postgres")] pg: copy_args.pg, }; diff --git a/martin/src/config.rs b/martin/src/config.rs index b8e2b5386..f311d5e69 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -11,15 +11,13 @@ use log::info; use serde::{Deserialize, Serialize}; use subst::VariableMap; -use crate::file_config::{resolve_files, resolve_files_urls, FileConfigEnum}; +#[cfg(any(feature = "mbtiles", feature = "pmtiles", feature = "sprites"))] +use crate::file_config::FileConfigEnum; #[cfg(feature = "fonts")] use crate::fonts::FontSources; -use crate::mbtiles::MbtilesConfig; -use crate::pg::PgConfig; -use crate::pmtiles::PmtConfig; use crate::source::{TileInfoSources, TileSources}; #[cfg(feature = "sprites")] -use crate::sprites::SpriteSources; +use crate::sprites::{SpriteConfig, SpriteSources}; use crate::srv::SrvConfig; use crate::MartinError::{ConfigLoadError, ConfigParseError, ConfigWriteError, NoSources}; use crate::{IdResolver, MartinResult, OptOneMany}; @@ -39,18 +37,21 @@ pub struct Config { #[serde(flatten)] pub srv: SrvConfig, + #[cfg(feature = "postgres")] #[serde(default, skip_serializing_if = "OptOneMany::is_none")] - pub postgres: OptOneMany, + pub postgres: OptOneMany, + #[cfg(feature = "pmtiles")] #[serde(default, skip_serializing_if = "FileConfigEnum::is_none")] - pub pmtiles: FileConfigEnum, + pub pmtiles: FileConfigEnum, + #[cfg(feature = "mbtiles")] #[serde(default, skip_serializing_if = "FileConfigEnum::is_none")] - pub mbtiles: FileConfigEnum, + pub mbtiles: FileConfigEnum, #[cfg(feature = "sprites")] #[serde(default, skip_serializing_if = "FileConfigEnum::is_none")] - pub sprites: FileConfigEnum, + pub sprites: FileConfigEnum, #[serde(default, skip_serializing_if = "OptOneMany::is_none")] pub fonts: OptOneMany, @@ -65,20 +66,33 @@ impl Config { let mut res = UnrecognizedValues::new(); copy_unrecognized_config(&mut res, "", &self.unrecognized); + #[cfg(feature = "postgres")] for pg in self.postgres.iter_mut() { res.extend(pg.finalize()?); } + #[cfg(feature = "pmtiles")] res.extend(self.pmtiles.finalize("pmtiles.")?); + + #[cfg(feature = "mbtiles")] res.extend(self.mbtiles.finalize("mbtiles.")?); + #[cfg(feature = "sprites")] res.extend(self.sprites.finalize("sprites.")?); // TODO: support for unrecognized fonts? // res.extend(self.fonts.finalize("fonts.")?); - let is_empty = - self.postgres.is_empty() && self.pmtiles.is_empty() && self.mbtiles.is_empty(); + let is_empty = true; + + #[cfg(feature = "postgres")] + let is_empty = is_empty && self.postgres.is_empty(); + + #[cfg(feature = "pmtiles")] + let is_empty = is_empty && self.pmtiles.is_empty(); + + #[cfg(feature = "mbtiles")] + let is_empty = is_empty && self.mbtiles.is_empty(); #[cfg(feature = "sprites")] let is_empty = is_empty && self.sprites.is_empty(); @@ -103,23 +117,30 @@ impl Config { }) } - async fn resolve_tile_sources(&mut self, idr: IdResolver) -> MartinResult { + async fn resolve_tile_sources( + &mut self, + #[allow(unused_variables)] idr: IdResolver, + ) -> MartinResult { + #[allow(unused_mut)] let mut sources: Vec>>>> = Vec::new(); + #[cfg(feature = "postgres")] for s in self.postgres.iter_mut() { sources.push(Box::pin(s.resolve(idr.clone()))); } + #[cfg(feature = "pmtiles")] if !self.pmtiles.is_empty() { let cfg = &mut self.pmtiles; - let val = resolve_files_urls(cfg, idr.clone(), "pmtiles"); + let val = crate::file_config::resolve_files(cfg, idr.clone(), "pmtiles"); sources.push(Box::pin(val)); } + #[cfg(feature = "mbtiles")] if !self.mbtiles.is_empty() { let cfg = &mut self.mbtiles; - let val = resolve_files(cfg, idr.clone(), "mbtiles"); + let val = crate::file_config::resolve_files(cfg, idr.clone(), "mbtiles"); sources.push(Box::pin(val)); } @@ -180,6 +201,7 @@ where subst::yaml::from_str(contents, env).map_err(|e| ConfigParseError(e, file_name.into())) } +#[cfg(feature = "postgres")] #[cfg(test)] pub mod tests { use super::*; diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index baa9d88af..3fdb847e8 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -40,32 +40,40 @@ pub enum FileError { #[error(r"Unable to parse metadata in file {1}: {0}")] InvalidUrlMetadata(String, Url), - #[error(r#"Unable to aquire connection to file: {0}"#)] - AquireConnError(String), + #[error(r#"Unable to acquire connection to file: {0}"#)] + AcquireConnError(String), #[error(r#"PMTiles error {0} processing {1}"#)] PmtError(pmtiles::PmtError, String), } +pub trait ConfigExtras: Clone + Debug + Default + PartialEq + Send { + fn init_parsing(&mut self) -> FileResult<()> { + Ok(()) + } + + #[must_use] + fn is_default(&self) -> bool { + true + } + + fn get_unrecognized(&self) -> &UnrecognizedValues; +} + #[async_trait] -pub trait FileConfigExtras: Clone + Debug + Default + PartialEq + Send { - fn parse_urls() -> bool; - async fn new_sources( - cfg: Option<&Self>, - id: String, - path: PathBuf, - ) -> FileResult>; - - async fn new_sources_url( - cfg: Option<&Self>, - id: String, - url: Url, - ) -> FileResult>; +pub trait SourceConfigExtras: ConfigExtras { + #[must_use] + fn parse_urls() -> bool { + false + } + async fn new_sources(&self, id: String, path: PathBuf) -> FileResult>; + + async fn new_sources_url(&self, id: String, url: Url) -> FileResult>; } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(untagged)] -pub enum FileConfigEnum { +pub enum FileConfigEnum { #[default] None, Path(PathBuf), @@ -73,20 +81,19 @@ pub enum FileConfigEnum { Config(FileConfig), } -impl FileConfigEnum { +impl FileConfigEnum { #[must_use] pub fn new(paths: Vec) -> FileConfigEnum { - Self::new_extended(paths, BTreeMap::new(), None, UnrecognizedValues::new()) + Self::new_extended(paths, BTreeMap::new(), T::default()) } #[must_use] pub fn new_extended( paths: Vec, configs: BTreeMap, - extras: Option, - unrecognized: UnrecognizedValues, + extras: T, ) -> Self { - if configs.is_empty() && extras.is_none() && unrecognized.is_empty() { + if configs.is_empty() && extras.is_default() { match paths.len() { 0 => FileConfigEnum::None, 1 => FileConfigEnum::Path(paths.into_iter().next().unwrap()), @@ -101,7 +108,6 @@ impl FileConfigEnum { Some(configs) }, extras, - unrecognized, }) } } @@ -121,25 +127,27 @@ impl FileConfigEnum { } } - pub fn extract_file_config(&mut self) -> Option> { - match self { - FileConfigEnum::None => None, - FileConfigEnum::Path(path) => Some(FileConfig { + pub fn extract_file_config(&mut self) -> FileResult>> { + let mut res = match self { + FileConfigEnum::None => return Ok(None), + FileConfigEnum::Path(path) => FileConfig { paths: One(mem::take(path)), ..FileConfig::default() - }), - FileConfigEnum::Paths(paths) => Some(FileConfig { + }, + FileConfigEnum::Paths(paths) => FileConfig { paths: Many(mem::take(paths)), ..Default::default() - }), - FileConfigEnum::Config(cfg) => Some(mem::take(cfg)), - } + }, + FileConfigEnum::Config(cfg) => mem::take(cfg), + }; + res.extras.init_parsing()?; + Ok(Some(res)) } pub fn finalize(&self, prefix: &str) -> MartinResult { let mut res = UnrecognizedValues::new(); if let Self::Config(cfg) = self { - copy_unrecognized_config(&mut res, prefix, &cfg.unrecognized); + copy_unrecognized_config(&mut res, prefix, cfg.get_unrecognized()); } Ok(res) } @@ -153,16 +161,22 @@ pub struct FileConfig { pub paths: OptOneMany, /// A map of source IDs to file paths or config objects pub sources: Option>, + /// Any customizations related to the specifics of the configuration section #[serde(flatten)] - pub extras: Option, - #[serde(flatten)] - pub unrecognized: UnrecognizedValues, + pub extras: T, } -impl FileConfig { +impl FileConfig { #[must_use] pub fn is_empty(&self) -> bool { - self.paths.is_none() && self.sources.is_none() + self.paths.is_none() + && self.sources.is_none() + && self.get_unrecognized().is_empty() + && self.extras.is_default() + } + + pub fn get_unrecognized(&self) -> &UnrecognizedValues { + self.extras.get_unrecognized() } } @@ -202,34 +216,22 @@ pub struct FileConfigSource { pub path: PathBuf, } -pub async fn resolve_files( - config: &mut FileConfigEnum, - idr: IdResolver, - extension: &str, -) -> MartinResult { - resolve_int(config, idr, extension) - .map_err(crate::MartinError::from) - .await -} - -pub async fn resolve_files_urls( +pub async fn resolve_files( config: &mut FileConfigEnum, idr: IdResolver, extension: &str, - // new_source: &mut impl FnMut(String, PathBuf) -> Fut1, - // new_url_source: &mut impl FnMut(String, Url) -> Fut2, ) -> MartinResult { resolve_int(config, idr, extension) .map_err(crate::MartinError::from) .await } -async fn resolve_int( +async fn resolve_int( config: &mut FileConfigEnum, idr: IdResolver, extension: &str, ) -> FileResult { - let Some(cfg) = config.extract_file_config() else { + let Some(cfg) = config.extract_file_config()? else { return Ok(TileInfoSources::default()); }; @@ -245,8 +247,7 @@ async fn resolve_int( let dup = if dup { "duplicate " } else { "" }; let id = idr.resolve(&id, url.to_string()); configs.insert(id.clone(), source); - results - .push(T::new_sources_url(cfg.extras.as_ref(), id.clone(), url.clone()).await?); + results.push(cfg.extras.new_sources_url(id.clone(), url.clone()).await?); info!("Configured {dup}source {id} from {}", sanitize_url(&url)); } else { let can = source.abs_path()?; @@ -260,7 +261,7 @@ async fn resolve_int( let id = idr.resolve(&id, can.to_string_lossy().to_string()); info!("Configured {dup}source {id} from {}", can.display()); configs.insert(id.clone(), source.clone()); - results.push(T::new_sources(cfg.extras.as_ref(), id, source.into_path()).await?); + results.push(cfg.extras.new_sources(id, source.into_path()).await?); } } } @@ -280,7 +281,7 @@ async fn resolve_int( let id = idr.resolve(id, url.to_string()); configs.insert(id.clone(), FileConfigSrc::Path(path)); - results.push(T::new_sources_url(cfg.extras.as_ref(), id.clone(), url.clone()).await?); + results.push(cfg.extras.new_sources_url(id.clone(), url.clone()).await?); info!("Configured source {id} from URL {}", sanitize_url(&url)); } else { let is_dir = path.is_dir(); @@ -309,12 +310,12 @@ async fn resolve_int( info!("Configured source {id} from {}", can.display()); files.insert(can); configs.insert(id.clone(), FileConfigSrc::Path(path.clone())); - results.push(T::new_sources(cfg.extras.as_ref(), id, path).await?); + results.push(cfg.extras.new_sources(id, path).await?); } } } - *config = FileConfigEnum::new_extended(directories, configs, cfg.extras, cfg.unrecognized); + *config = FileConfigEnum::new_extended(directories, configs, cfg.extras); Ok(results) } @@ -353,71 +354,3 @@ fn parse_url(is_enabled: bool, path: &Path) -> Result, FileError> { .map(|v| Url::parse(v).map_err(|e| InvalidSourceUrl(e, v.to_string()))) .transpose() } - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::path::PathBuf; - - use indoc::indoc; - - use crate::file_config::{FileConfigEnum, FileConfigSource, FileConfigSrc}; - use crate::mbtiles::MbtilesConfig; - - #[test] - fn parse() { - let cfg = serde_yaml::from_str::>(indoc! {" - paths: - - /dir-path - - /path/to/file2.ext - - http://example.org/file.ext - sources: - pm-src1: /tmp/file.ext - pm-src2: - path: /tmp/file.ext - pm-src3: https://example.org/file3.ext - pm-src4: - path: https://example.org/file4.ext - "}) - .unwrap(); - let res = cfg.finalize("").unwrap(); - assert!(res.is_empty(), "unrecognized config: {res:?}"); - let FileConfigEnum::Config(cfg) = cfg else { - panic!(); - }; - let paths = cfg.paths.clone().into_iter().collect::>(); - assert_eq!( - paths, - vec![ - PathBuf::from("/dir-path"), - PathBuf::from("/path/to/file2.ext"), - PathBuf::from("http://example.org/file.ext"), - ] - ); - assert_eq!( - cfg.sources, - Some(BTreeMap::from_iter(vec![ - ( - "pm-src1".to_string(), - FileConfigSrc::Path(PathBuf::from("/tmp/file.ext")) - ), - ( - "pm-src2".to_string(), - FileConfigSrc::Obj(FileConfigSource { - path: PathBuf::from("/tmp/file.ext"), - }) - ), - ( - "pm-src3".to_string(), - FileConfigSrc::Path(PathBuf::from("https://example.org/file3.ext")) - ), - ( - "pm-src4".to_string(), - FileConfigSrc::Obj(FileConfigSource { - path: PathBuf::from("https://example.org/file4.ext"), - }) - ), - ])) - ); - } -} diff --git a/martin/src/lib.rs b/martin/src/lib.rs index 56b15358b..c0c9d4728 100644 --- a/martin/src/lib.rs +++ b/martin/src/lib.rs @@ -17,8 +17,11 @@ pub mod args; pub mod file_config; #[cfg(feature = "fonts")] pub mod fonts; +#[cfg(feature = "mbtiles")] pub mod mbtiles; +#[cfg(feature = "postgres")] pub mod pg; +#[cfg(feature = "pmtiles")] pub mod pmtiles; #[cfg(feature = "sprites")] pub mod sprites; diff --git a/martin/src/mbtiles/mod.rs b/martin/src/mbtiles/mod.rs index 2b260ca15..756046b47 100644 --- a/martin/src/mbtiles/mod.rs +++ b/martin/src/mbtiles/mod.rs @@ -11,33 +11,31 @@ use serde::{Deserialize, Serialize}; use tilejson::TileJSON; use url::Url; -use crate::file_config::FileError::{AquireConnError, InvalidMetadata, IoError}; -use crate::file_config::{FileConfigExtras, FileResult}; +use crate::config::UnrecognizedValues; +use crate::file_config::FileError::{AcquireConnError, InvalidMetadata, IoError}; +use crate::file_config::{ConfigExtras, FileResult, SourceConfigExtras}; use crate::source::{TileData, UrlQuery}; use crate::{MartinResult, Source, TileCoord}; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct MbtilesConfig; +pub struct MbtConfig { + #[serde(flatten)] + pub unrecognized: UnrecognizedValues, +} -#[async_trait] -impl FileConfigExtras for MbtilesConfig { - async fn new_sources( - _cfg: Option<&Self>, - id: String, - path: PathBuf, - ) -> FileResult> { - Ok(Box::new(MbtSource::new(id, path).await?)) +impl ConfigExtras for MbtConfig { + fn get_unrecognized(&self) -> &UnrecognizedValues { + &self.unrecognized } +} - fn parse_urls() -> bool { - false +#[async_trait] +impl SourceConfigExtras for MbtConfig { + async fn new_sources(&self, id: String, path: PathBuf) -> FileResult> { + Ok(Box::new(MbtSource::new(id, path).await?)) } - async fn new_sources_url( - _cfg: Option<&Self>, - _id: String, - _url: Url, - ) -> FileResult> { + async fn new_sources_url(&self, _id: String, _url: Url) -> FileResult> { unreachable!() } } @@ -114,7 +112,7 @@ impl Source for MbtSource { .mbtiles .get_tile(xyz.z, xyz.x, xyz.y) .await - .map_err(|_| AquireConnError(self.id.clone()))? + .map_err(|_| AcquireConnError(self.id.clone()))? { Ok(tile) } else { @@ -129,3 +127,71 @@ impl Source for MbtSource { } } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::path::PathBuf; + + use indoc::indoc; + + use crate::file_config::{FileConfigEnum, FileConfigSource, FileConfigSrc}; + use crate::mbtiles::MbtConfig; + + #[test] + fn parse() { + let cfg = serde_yaml::from_str::>(indoc! {" + paths: + - /dir-path + - /path/to/file2.ext + - http://example.org/file.ext + sources: + pm-src1: /tmp/file.ext + pm-src2: + path: /tmp/file.ext + pm-src3: https://example.org/file3.ext + pm-src4: + path: https://example.org/file4.ext + "}) + .unwrap(); + let res = cfg.finalize("").unwrap(); + assert!(res.is_empty(), "unrecognized config: {res:?}"); + let FileConfigEnum::Config(cfg) = cfg else { + panic!(); + }; + let paths = cfg.paths.clone().into_iter().collect::>(); + assert_eq!( + paths, + vec![ + PathBuf::from("/dir-path"), + PathBuf::from("/path/to/file2.ext"), + PathBuf::from("http://example.org/file.ext"), + ] + ); + assert_eq!( + cfg.sources, + Some(BTreeMap::from_iter(vec![ + ( + "pm-src1".to_string(), + FileConfigSrc::Path(PathBuf::from("/tmp/file.ext")) + ), + ( + "pm-src2".to_string(), + FileConfigSrc::Obj(FileConfigSource { + path: PathBuf::from("/tmp/file.ext"), + }) + ), + ( + "pm-src3".to_string(), + FileConfigSrc::Path(PathBuf::from("https://example.org/file3.ext")) + ), + ( + "pm-src4".to_string(), + FileConfigSrc::Obj(FileConfigSource { + path: PathBuf::from("https://example.org/file4.ext"), + }) + ), + ])) + ); + } +} diff --git a/martin/src/pg/config.rs b/martin/src/pg/config.rs index 4f44aa51a..c3900200c 100644 --- a/martin/src/pg/config.rs +++ b/martin/src/pg/config.rs @@ -11,9 +11,10 @@ use crate::config::{copy_unrecognized_config, UnrecognizedValues}; use crate::pg::config_function::FuncInfoSources; use crate::pg::config_table::TableInfoSources; use crate::pg::configurator::PgBuilder; +use crate::pg::utils::on_slow; use crate::pg::PgResult; use crate::source::TileInfoSources; -use crate::utils::{on_slow, IdResolver, OptBoolObj, OptOneMany}; +use crate::utils::{IdResolver, OptBoolObj, OptOneMany}; use crate::MartinResult; pub trait PgInfo { diff --git a/martin/src/pg/configurator.rs b/martin/src/pg/configurator.rs index 9e0bde7a5..bbd8384c2 100644 --- a/martin/src/pg/configurator.rs +++ b/martin/src/pg/configurator.rs @@ -36,7 +36,7 @@ pub struct PgBuilderFuncs { #[derive(Debug, Default, PartialEq)] #[cfg_attr(test, serde_with::skip_serializing_none, derive(serde::Serialize))] pub struct PgBuilderTables { - #[cfg_attr(test, serde(serialize_with = "crate::utils::sorted_opt_set"))] + #[cfg_attr(test, serde(serialize_with = "crate::pg::utils::sorted_opt_set"))] schemas: Option>, source_id_format: String, id_columns: Option>, diff --git a/martin/src/pg/utils.rs b/martin/src/pg/utils.rs index 8d79c214a..7733d9b61 100755 --- a/martin/src/pg/utils.rs +++ b/martin/src/pg/utils.rs @@ -1,13 +1,48 @@ use std::collections::{BTreeMap, HashMap}; +use std::future::Future; +use std::time::Duration; use deadpool_postgres::tokio_postgres::types::Json; +use futures::pin_mut; use itertools::Itertools as _; use log::{error, info, warn}; use postgis::{ewkb, LineString, Point, Polygon}; use tilejson::{Bounds, TileJSON}; +use tokio::time::timeout; use crate::source::UrlQuery; +#[cfg(test)] +pub fn sorted_opt_set( + value: &Option>, + serializer: S, +) -> Result { + use serde::Serialize as _; + + value + .as_ref() + .map(|v| { + let mut v: Vec<_> = v.iter().collect(); + v.sort(); + v + }) + .serialize(serializer) +} + +pub async fn on_slow( + future: impl Future, + duration: Duration, + fn_on_slow: S, +) -> T { + pin_mut!(future); + if let Ok(result) = timeout(duration, &mut future).await { + result + } else { + fn_on_slow(); + future.await + } +} + #[must_use] pub fn json_to_hashmap(value: &serde_json::Value) -> InfoMap { let mut result = BTreeMap::new(); diff --git a/martin/src/pmtiles/file_pmtiles.rs b/martin/src/pmtiles/file_pmtiles.rs new file mode 100644 index 000000000..9f88489a6 --- /dev/null +++ b/martin/src/pmtiles/file_pmtiles.rs @@ -0,0 +1,58 @@ +use std::fmt::{Debug, Formatter}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use log::{trace, warn}; +use martin_tile_utils::{Encoding, Format, TileInfo}; +use pmtiles::async_reader::AsyncPmTilesReader; +use pmtiles::cache::NoCache; +use pmtiles::mmap::MmapBackend; +use pmtiles::{Compression, TileType}; +use tilejson::TileJSON; + +use crate::file_config::FileError::{InvalidMetadata, IoError}; +use crate::file_config::FileResult; +use crate::pmtiles::impl_pmtiles_source; +use crate::source::{Source, UrlQuery}; +use crate::{MartinResult, TileCoord, TileData}; + +impl_pmtiles_source!( + PmtFileSource, + MmapBackend, + NoCache, + PathBuf, + Path::display, + InvalidMetadata +); + +impl PmtFileSource { + pub async fn new_box(id: String, path: PathBuf) -> FileResult> { + Ok(Box::new(PmtFileSource::new(id, path).await?)) + } + + pub async fn new(id: String, path: PathBuf) -> FileResult { + let backend = MmapBackend::try_from(path.as_path()) + .await + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("{e:?}: Cannot open file {}", path.display()), + ) + }) + .map_err(|e| IoError(e, path.clone()))?; + + let reader = AsyncPmTilesReader::try_from_source(backend).await; + let reader = reader + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("{e:?}: Cannot open file {}", path.display()), + ) + }) + .map_err(|e| IoError(e, path.clone()))?; + + Self::new_int(id, path, reader).await + } +} diff --git a/martin/src/pmtiles/http_pmtiles.rs b/martin/src/pmtiles/http_pmtiles.rs new file mode 100644 index 000000000..6b806a290 --- /dev/null +++ b/martin/src/pmtiles/http_pmtiles.rs @@ -0,0 +1,42 @@ +use std::convert::identity; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +use async_trait::async_trait; +use log::{trace, warn}; +use martin_tile_utils::{Encoding, Format, TileInfo}; +use pmtiles::async_reader::AsyncPmTilesReader; +use pmtiles::http::HttpBackend; +use pmtiles::{Compression, TileType}; +use reqwest::Client; +use tilejson::TileJSON; +use url::Url; + +use crate::file_config::FileError::InvalidUrlMetadata; +use crate::file_config::{FileError, FileResult}; +use crate::pmtiles::{impl_pmtiles_source, PmtCache}; +use crate::source::{Source, UrlQuery}; +use crate::{MartinResult, TileCoord, TileData}; + +impl_pmtiles_source!( + PmtHttpSource, + HttpBackend, + PmtCache, + Url, + identity, + InvalidUrlMetadata +); + +impl PmtHttpSource { + pub async fn new_url( + client: Client, + cache: PmtCache, + id: String, + url: Url, + ) -> FileResult { + let reader = AsyncPmTilesReader::new_with_cached_url(cache, client, url.clone()).await; + let reader = reader.map_err(|e| FileError::PmtError(e, url.to_string()))?; + + Self::new_int(id, url, reader).await + } +} diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index 91201f520..de5c08941 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -1,32 +1,147 @@ -use std::convert::identity; -use std::fmt::{Debug, Formatter}; -use std::io; -use std::path::{Path, PathBuf}; -use std::sync::Arc; +mod file_pmtiles; +mod http_pmtiles; + +use std::path::PathBuf; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::Relaxed; use async_trait::async_trait; -use log::{trace, warn}; -use martin_tile_utils::{Encoding, Format, TileInfo}; +pub use file_pmtiles::PmtFileSource; +pub use http_pmtiles::PmtHttpSource; use moka::future::Cache; -use pmtiles::async_reader::AsyncPmTilesReader; -use pmtiles::cache::{DirCacheResult, DirectoryCache, NoCache}; -use pmtiles::http::HttpBackend; -use pmtiles::mmap::MmapBackend; -use pmtiles::{Compression, Directory, TileType}; +use pmtiles::cache::{DirCacheResult, DirectoryCache}; +use pmtiles::Directory; use reqwest::Client; use serde::{Deserialize, Serialize}; -use tilejson::TileJSON; use url::Url; -use crate::file_config::FileError::{InvalidMetadata, InvalidUrlMetadata, IoError}; -use crate::file_config::{FileConfigExtras, FileError, FileResult}; -use crate::source::{Source, UrlQuery}; -use crate::{MartinResult, TileCoord, TileData}; +use crate::file_config::{ConfigExtras, FileResult, SourceConfigExtras}; +use crate::Source; + +type PmtCacheObject = Cache<(usize, usize), Directory>; + +#[derive(Clone, Debug)] +pub struct PmtCache { + id: usize, + /// (id, offset) -> Directory, or None to disable caching + cache: Option, +} + +impl PmtCache { + #[must_use] + pub fn new(id: usize, cache: Option) -> Self { + Self { id, cache } + } +} + +#[async_trait] +impl DirectoryCache for PmtCache { + async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { + if let Some(cache) = &self.cache { + if let Some(dir) = cache.get(&(self.id, offset)).await { + return dir.find_tile_id(tile_id).into(); + } + } + DirCacheResult::NotCached + } + + async fn insert_dir(&self, offset: usize, directory: Directory) { + if let Some(cache) = &self.cache { + cache.insert((self.id, offset), directory).await; + } + } +} #[serde_with::skip_serializing_none] -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Default, Serialize, Deserialize)] pub struct PmtConfig { - pub dir_cache_size: Option, + pub dir_cache_size_mb: Option, + + #[serde(flatten)] + pub unrecognized: UnrecognizedValues, + + // + // The rest are internal state, not serialized + // + #[serde(skip)] + pub client: Option, + + #[serde(skip)] + pub next_cache_id: AtomicUsize, + + #[serde(skip)] + pub cache: Option, +} + +impl PmtConfig { + pub fn new_cache_user(&self) -> PmtCache { + PmtCache::new(self.next_cache_id.fetch_add(1, Relaxed), self.cache.clone()) + } +} + +impl Clone for PmtConfig { + fn clone(&self) -> Self { + // State is not shared between clones, only the serialized config + Self { + dir_cache_size_mb: self.dir_cache_size_mb, + ..Default::default() + } + } +} + +impl ConfigExtras for PmtConfig { + fn init_parsing(&mut self) -> FileResult<()> { + assert!(self.client.is_none()); + assert!(self.cache.is_none()); + + self.client = Some(Client::new()); + + // Allow cache size to be disabled with 0 + let cache_size = self.dir_cache_size_mb.unwrap_or(32) * 1024 * 1024; + if cache_size > 0 { + self.cache = Some( + Cache::builder() + .weigher(|_key, value: &Directory| -> u32 { + value.get_approx_byte_size().try_into().unwrap_or(u32::MAX) + }) + .max_capacity(cache_size) + .build(), + ); + } + Ok(()) + } + + fn is_default(&self) -> bool { + true + } + + fn get_unrecognized(&self) -> &UnrecognizedValues { + &self.unrecognized + } +} + +#[async_trait] +impl SourceConfigExtras for PmtConfig { + fn parse_urls() -> bool { + true + } + + async fn new_sources(&self, id: String, path: PathBuf) -> FileResult> { + Ok(Box::new(PmtFileSource::new(id, path).await?)) + } + + async fn new_sources_url(&self, id: String, url: Url) -> FileResult> { + Ok(Box::new( + PmtHttpSource::new_url(self.client.clone().unwrap(), self.new_cache_user(), id, url) + .await?, + )) + } +} + +impl PartialEq for PmtConfig { + fn eq(&self, other: &Self) -> bool { + self.dir_cache_size_mb == other.dir_cache_size_mb + } } macro_rules! impl_pmtiles_source { @@ -157,113 +272,6 @@ macro_rules! impl_pmtiles_source { }; } -impl_pmtiles_source!( - PmtFileSource, - MmapBackend, - NoCache, - PathBuf, - Path::display, - InvalidMetadata -); - -#[async_trait] -impl FileConfigExtras for PmtConfig { - fn parse_urls() -> bool { - true - } - - async fn new_sources( - _cfg: Option<&Self>, - id: String, - path: PathBuf, - ) -> FileResult> { - Ok(Box::new(PmtFileSource::new(id, path).await?)) - - // let client = Client::new(); - // let cache = PmtCache::new(4 * 1024 * 1024); - // Ok(Box::new( - // PmtHttpSource::new_url(client, cache, id, url).await?, - // )) - } - - async fn new_sources_url( - _cfg: Option<&Self>, - _id: String, - _url: Url, - ) -> FileResult> { - unreachable!() - } -} - -impl PmtFileSource { - async fn new(id: String, path: PathBuf) -> FileResult { - let backend = MmapBackend::try_from(path.as_path()) - .await - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) - .map_err(|e| IoError(e, path.clone()))?; - - let reader = AsyncPmTilesReader::try_from_source(backend).await; - let reader = reader - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) - .map_err(|e| IoError(e, path.clone()))?; - - Self::new_int(id, path, reader).await - } -} - -struct PmtCache(Cache); +pub(crate) use impl_pmtiles_source; -impl PmtCache { - fn new(max_capacity: u64) -> Self { - Self( - Cache::builder() - .weigher(|_key, value: &Directory| -> u32 { - value.get_approx_byte_size().try_into().unwrap_or(u32::MAX) - }) - .max_capacity(max_capacity) - .build(), - ) - } -} - -#[async_trait] -impl DirectoryCache for PmtCache { - async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { - match self.0.get(&offset).await { - Some(dir) => dir.find_tile_id(tile_id).into(), - None => DirCacheResult::NotCached, - } - } - - async fn insert_dir(&self, offset: usize, directory: Directory) { - self.0.insert(offset, directory).await; - } -} - -impl_pmtiles_source!( - PmtHttpSource, - HttpBackend, - PmtCache, - Url, - identity, - InvalidUrlMetadata -); - -impl PmtHttpSource { - async fn new_url(client: Client, cache: PmtCache, id: String, url: Url) -> FileResult { - let reader = AsyncPmTilesReader::new_with_cached_url(cache, client, url.clone()).await; - let reader = reader.map_err(|e| FileError::PmtError(e, url.to_string()))?; - - Self::new_int(id, url, reader).await - } -} +use crate::config::UnrecognizedValues; diff --git a/martin/src/sprites/mod.rs b/martin/src/sprites/mod.rs index 3a7daa666..be1af14fe 100644 --- a/martin/src/sprites/mod.rs +++ b/martin/src/sprites/mod.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, HashMap}; use std::fmt::Debug; use std::path::PathBuf; +use async_trait::async_trait; use futures::future::try_join_all; use log::{info, warn}; use serde::{Deserialize, Serialize}; @@ -13,7 +14,8 @@ use spreet::{ use tokio::io::AsyncReadExt; use self::SpriteError::{SpriteInstError, SpriteParsingError, SpriteProcessingError}; -use crate::file_config::{FileConfigEnum, FileResult}; +use crate::config::UnrecognizedValues; +use crate::file_config::{ConfigExtras, FileConfigEnum, FileResult}; pub type SpriteResult = Result; @@ -57,12 +59,25 @@ pub struct CatalogSpriteEntry { pub type SpriteCatalog = BTreeMap; +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct SpriteConfig { + #[serde(flatten)] + pub unrecognized: UnrecognizedValues, +} + +#[async_trait] +impl ConfigExtras for SpriteConfig { + fn get_unrecognized(&self) -> &UnrecognizedValues { + &self.unrecognized + } +} + #[derive(Debug, Clone, Default)] pub struct SpriteSources(HashMap); impl SpriteSources { - pub fn resolve(config: &mut FileConfigEnum) -> FileResult { - let Some(cfg) = config.extract_file_config() else { + pub fn resolve(config: &mut FileConfigEnum) -> FileResult { + let Some(cfg) = config.extract_file_config()? else { return Ok(Self::default()); }; @@ -89,7 +104,7 @@ impl SpriteSources { results.add_source(name.to_string_lossy().to_string(), path); } - *config = FileConfigEnum::new_extended(directories, configs, cfg.extras, cfg.unrecognized); + *config = FileConfigEnum::new_extended(directories, configs, cfg.extras); Ok(results) } diff --git a/martin/src/utils/error.rs b/martin/src/utils/error.rs index e4a68e5b6..4d80ebe74 100644 --- a/martin/src/utils/error.rs +++ b/martin/src/utils/error.rs @@ -8,7 +8,6 @@ use mbtiles::MbtError; use crate::file_config::FileError; #[cfg(feature = "fonts")] use crate::fonts::FontError; -use crate::pg::PgError; #[cfg(feature = "sprites")] use crate::sprites::SpriteError; @@ -58,8 +57,9 @@ pub enum MartinError { #[error("Unrecognizable connection strings: {0:?}")] UnrecognizableConnections(Vec), + #[cfg(feature = "postgres")] #[error(transparent)] - PostgresError(#[from] PgError), + PostgresError(#[from] crate::pg::PgError), #[error(transparent)] MbtilesError(#[from] MbtError), diff --git a/martin/src/utils/utilities.rs b/martin/src/utils/utilities.rs index ac4e68b62..1bdb177ba 100644 --- a/martin/src/utils/utilities.rs +++ b/martin/src/utils/utilities.rs @@ -1,28 +1,7 @@ -use std::future::Future; use std::io::{Read as _, Write as _}; -use std::time::Duration; use flate2::read::GzDecoder; use flate2::write::GzEncoder; -use futures::pin_mut; -use tokio::time::timeout; - -#[cfg(test)] -pub fn sorted_opt_set( - value: &Option>, - serializer: S, -) -> Result { - use serde::Serialize as _; - - value - .as_ref() - .map(|v| { - let mut v: Vec<_> = v.iter().collect(); - v.sort(); - v - }) - .serialize(serializer) -} pub fn decode_gzip(data: &[u8]) -> Result, std::io::Error> { let mut decoder = GzDecoder::new(data); @@ -49,17 +28,3 @@ pub fn encode_brotli(data: &[u8]) -> Result, std::io::Error> { encoder.write_all(data)?; Ok(encoder.into_inner()) } - -pub async fn on_slow( - future: impl Future, - duration: Duration, - fn_on_slow: S, -) -> T { - pin_mut!(future); - if let Ok(result) = timeout(duration, &mut future).await { - result - } else { - fn_on_slow(); - future.await - } -} diff --git a/martin/tests/pg_function_source_test.rs b/martin/tests/pg_function_source_test.rs index 3e00ef62d..3bf47300d 100644 --- a/martin/tests/pg_function_source_test.rs +++ b/martin/tests/pg_function_source_test.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "postgres")] + use ctor::ctor; use indoc::indoc; use insta::assert_yaml_snapshot; diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index 4757b783c..fb0be8c22 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "postgres")] + use actix_http::Request; use actix_web::http::StatusCode; use actix_web::test::{call_and_read_body_json, call_service, read_body, TestRequest}; diff --git a/martin/tests/pg_table_source_test.rs b/martin/tests/pg_table_source_test.rs index b82e990e3..8def85309 100644 --- a/martin/tests/pg_table_source_test.rs +++ b/martin/tests/pg_table_source_test.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "postgres")] + use ctor::ctor; use indoc::indoc; use insta::assert_yaml_snapshot; diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs index 27b689420..334b1179f 100644 --- a/martin/tests/utils/pg_utils.rs +++ b/martin/tests/utils/pg_utils.rs @@ -1,6 +1,5 @@ use indoc::formatdoc; pub use martin::args::Env; -use martin::pg::TableInfo; use martin::{Config, IdResolver, ServerState, Source}; use crate::mock_cfg; @@ -28,11 +27,12 @@ pub async fn mock_sources(mut config: Config) -> MockSource { (res, config) } +#[cfg(feature = "postgres")] #[allow(dead_code)] #[must_use] -pub fn table<'a>(mock: &'a MockSource, name: &str) -> &'a TableInfo { +pub fn table<'a>(mock: &'a MockSource, name: &str) -> &'a martin::pg::TableInfo { let (_, config) = mock; - let vals: Vec<&TableInfo> = config + let vals: Vec<&martin::pg::TableInfo> = config .postgres .iter() .flat_map(|v| v.tables.iter().map(|vv| vv.get(name))) diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index 8c6f44aab..f30213609 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.8.5" +version = "0.9.0" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/tests/config.yaml b/tests/config.yaml index 34649f40d..7a3f28488 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -166,7 +166,7 @@ postgres: pmtiles: - dir_cache_size: 100 + dir_cache_size_mb: 100 paths: - http://localhost:5412/webp2.pmtiles sources: diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 113e4d230..47379599a 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -165,7 +165,7 @@ pmtiles: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles pmt2: http://localhost:5412/webp2.pmtiles webp2: http://localhost:5412/webp2.pmtiles - dir_cache_size: 100 + dir_cache_size_mb: 100 sprites: paths: tests/fixtures/sprites/src1 sources: