diff --git a/Cargo.lock b/Cargo.lock index 222c2624e..c1f573960 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1166,6 +1166,21 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", + "serde", +] + [[package]] name = "data-url" version = "0.3.1" @@ -2667,6 +2682,7 @@ dependencies = [ "clap", "criterion", "ctor", + "dashmap", "deadpool-postgres", "enum-display", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index 96451cdc3..ebb8c8453 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ cargo-husky = { version = "1", features = ["user-hooks"], default-features = fal clap = { version = "4", features = ["derive"] } criterion = { version = "0.5", features = ["async_futures", "async_tokio", "html_reports"] } ctor = "0.2" +dashmap = { version = "6.1.0", features = ["serde", "inline", "rayon"] } deadpool-postgres = "0.14" enum-display = "0.1" env_logger = "0.11" diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 28dc1c4d8..1a9e522f2 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -79,6 +79,7 @@ async-trait.workspace = true bit-set = { workspace = true, optional = true } brotli.workspace = true clap.workspace = true +dashmap.workspace = true deadpool-postgres = { workspace = true, optional = true } enum-display.workspace = true env_logger.workspace = true diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 897bc72ad..2edddf4f4 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -392,7 +392,7 @@ fn parse_encoding(encoding: &str) -> MartinCpResult { async fn init_schema( mbt: &Mbtiles, conn: &mut SqliteConnection, - sources: &[&dyn Source], + sources: &[TileInfoSource], tile_info: TileInfo, args: &CopyArgs, ) -> Result { diff --git a/martin/src/fonts/mod.rs b/martin/src/fonts/mod.rs index 7ab657844..70390f346 100644 --- a/martin/src/fonts/mod.rs +++ b/martin/src/fonts/mod.rs @@ -1,11 +1,10 @@ -use std::collections::hash_map::Entry; -use std::collections::{BTreeMap, HashMap}; use std::ffi::OsStr; use std::fmt::Debug; use std::path::PathBuf; use std::sync::OnceLock; use bit_set::BitSet; +use dashmap::{DashMap, Entry}; use itertools::Itertools as _; use log::{debug, info, warn}; use pbf_font_tools::freetype::{Face, Library}; @@ -103,11 +102,11 @@ fn get_available_codepoints(face: &mut Face) -> Option { #[derive(Debug, Clone, Default)] pub struct FontSources { - fonts: HashMap, + fonts: DashMap, masks: Vec, } -pub type FontCatalog = BTreeMap; +pub type FontCatalog = DashMap; #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -125,7 +124,7 @@ impl FontSources { return Ok(Self::default()); } - let mut fonts = HashMap::new(); + let mut fonts = DashMap::new(); let lib = Library::init()?; for path in config.iter() { @@ -150,7 +149,7 @@ impl FontSources { pub fn get_catalog(&self) -> FontCatalog { self.fonts .iter() - .map(|(k, v)| (k.clone(), v.catalog_entry.clone())) + .map(|v| (v.key().clone(), v.catalog_entry.clone())) .sorted_by(|(a, _), (b, _)| a.cmp(b)) .collect() } @@ -242,7 +241,7 @@ pub struct FontSource { fn recurse_dirs( lib: &Library, path: PathBuf, - fonts: &mut HashMap, + fonts: &mut DashMap, is_top_level: bool, ) -> FontResult<()> { let start_count = fonts.len(); @@ -275,7 +274,7 @@ fn recurse_dirs( fn parse_font( lib: &Library, - fonts: &mut HashMap, + fonts: &mut DashMap, path: PathBuf, ) -> FontResult<()> { static RE_SPACES: OnceLock = OnceLock::new(); diff --git a/martin/src/source.rs b/martin/src/source.rs index d95f2fd51..e0e8d2bb6 100644 --- a/martin/src/source.rs +++ b/martin/src/source.rs @@ -1,8 +1,9 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::fmt::Debug; use actix_web::error::ErrorNotFound; use async_trait::async_trait; +use dashmap::DashMap; use log::debug; use martin_tile_utils::{TileCoord, TileInfo}; use serde::{Deserialize, Serialize}; @@ -18,8 +19,8 @@ pub type TileInfoSource = Box; pub type TileInfoSources = Vec; #[derive(Default, Clone)] -pub struct TileSources(HashMap); -pub type TileCatalog = BTreeMap; +pub struct TileSources(DashMap); +pub type TileCatalog = DashMap; impl TileSources { #[must_use] @@ -37,16 +38,17 @@ impl TileSources { pub fn get_catalog(&self) -> TileCatalog { self.0 .iter() - .map(|(id, src)| (id.to_string(), src.get_catalog_entry())) + .map(|v| (v.key().to_string(), v.get_catalog_entry())) .collect() } - pub fn get_source(&self, id: &str) -> actix_web::Result<&dyn Source> { + pub fn get_source(&self, id: &str) -> actix_web::Result { Ok(self .0 .get(id) .ok_or_else(|| ErrorNotFound(format!("Source {id} does not exist")))? - .as_ref()) + .value() + .clone()) } /// Get a list of sources, and the tile info for the merged sources. @@ -56,7 +58,7 @@ impl TileSources { &self, source_ids: &str, zoom: Option, - ) -> actix_web::Result<(Vec<&dyn Source>, bool, TileInfo)> { + ) -> actix_web::Result<(Vec, bool, TileInfo)> { let mut sources = Vec::new(); let mut info: Option = None; let mut use_url_query = false; @@ -78,7 +80,7 @@ impl TileSources { // TODO: Use chained-if-let once available if match zoom { - Some(zoom) if Self::check_zoom(src, id, zoom) => true, + Some(zoom) if Self::check_zoom(&*src, id, zoom) => true, None => true, _ => false, } { diff --git a/martin/src/sprites/mod.rs b/martin/src/sprites/mod.rs index 7817518b6..6a6067ac7 100644 --- a/martin/src/sprites/mod.rs +++ b/martin/src/sprites/mod.rs @@ -1,8 +1,4 @@ -use std::collections::hash_map::Entry; -use std::collections::{BTreeMap, HashMap}; -use std::fmt::Debug; -use std::path::PathBuf; - +use dashmap::{DashMap, Entry}; use futures::future::try_join_all; use log::{info, warn}; use serde::{Deserialize, Serialize}; @@ -10,6 +6,9 @@ use spreet::resvg::usvg::{Error as ResvgError, Options, Tree, TreeParsing}; use spreet::{ get_svg_input_paths, sprite_name, SpreetError, Sprite, Spritesheet, SpritesheetBuilder, }; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::path::PathBuf; use tokio::io::AsyncReadExt; use self::SpriteError::{SpriteInstError, SpriteParsingError, SpriteProcessingError}; @@ -56,7 +55,7 @@ pub struct CatalogSpriteEntry { pub images: Vec, } -pub type SpriteCatalog = BTreeMap; +pub type SpriteCatalog = DashMap; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct SpriteConfig { @@ -71,7 +70,7 @@ impl ConfigExtras for SpriteConfig { } #[derive(Debug, Clone, Default)] -pub struct SpriteSources(HashMap); +pub struct SpriteSources(DashMap); impl SpriteSources { pub fn resolve(config: &mut FileConfigEnum) -> FileResult { @@ -109,8 +108,8 @@ impl SpriteSources { pub fn get_catalog(&self) -> SpriteResult { // TODO: all sprite generation should be pre-cached - let mut entries = SpriteCatalog::new(); - for (id, source) in &self.0 { + let entries = SpriteCatalog::new(); + for source in &self.0 { let paths = get_svg_input_paths(&source.path, true) .map_err(|e| SpriteProcessingError(e, source.path.clone()))?; let mut images = Vec::with_capacity(paths.len()); @@ -121,7 +120,7 @@ impl SpriteSources { ); } images.sort(); - entries.insert(id.clone(), CatalogSpriteEntry { images }); + entries.insert(source.key().clone(), CatalogSpriteEntry { images }); } Ok(entries) } @@ -141,7 +140,7 @@ impl SpriteSources { v.insert(SpriteSource { path }); } } - }; + } } /// Given a list of IDs in a format "id1,id2,id3", return a spritesheet with them all. @@ -155,14 +154,17 @@ impl SpriteSources { let sprite_ids = ids .split(',') - .map(|id| { - self.0 - .get(id) - .ok_or_else(|| SpriteError::SpriteNotFound(id.to_string())) - }) + .map(|id| self.get(id)) .collect::>>()?; - get_spritesheet(sprite_ids.into_iter(), dpi, as_sdf).await + get_spritesheet(sprite_ids.iter(), dpi, as_sdf).await + } + + fn get(&self, id: &str) -> SpriteResult { + match self.0.get(id) { + Some(v) => Ok(v.clone()), + None => Err(SpriteError::SpriteNotFound(id.to_string())), + } } } @@ -247,19 +249,30 @@ mod tests { //.sdf => generate sdf from png, add sdf == true //- => does not generate sdf, omits sdf == true for extension in ["_sdf", ""] { - test_src(sprites.values(), 1, "all_1", extension).await; - test_src(sprites.values(), 2, "all_2", extension).await; - - test_src(sprites.get("src1").into_iter(), 1, "src1_1", extension).await; - test_src(sprites.get("src1").into_iter(), 2, "src1_2", extension).await; - - test_src(sprites.get("src2").into_iter(), 1, "src2_1", extension).await; - test_src(sprites.get("src2").into_iter(), 2, "src2_2", extension).await; + let paths = sprites.iter().map(|v| v.path.clone()).collect::>(); + test_src(paths.iter(), 1, "all_1", extension).await; + test_src(paths.iter(), 2, "all_2", extension).await; + + let src1_path = sprites + .get("src1") + .into_iter() + .map(|v| v.path.clone()) + .collect::>(); + test_src(src1_path.iter(), 1, "src1_1", extension).await; + test_src(src1_path.iter(), 2, "src1_2", extension).await; + + let src2_path = sprites + .get("src2") + .into_iter() + .map(|v| v.path.clone()) + .collect::>(); + test_src(src2_path.iter(), 1, "src2_1", extension).await; + test_src(src2_path.iter(), 2, "src2_2", extension).await; } } async fn test_src( - sources: impl Iterator, + sources: impl Iterator, pixel_ratio: u8, filename: &str, extension: &str, diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index a33d39a4f..7482e3ba3 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -40,7 +40,7 @@ pub const RESERVED_KEYWORDS: &[&str] = &[ "reload", "sprite", "status", ]; -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct Catalog { pub tiles: TileCatalog, #[cfg(feature = "sprites")] @@ -226,7 +226,7 @@ pub mod tests { } fn clone_source(&self) -> TileInfoSource { - unimplemented!() + Box::new(self.clone()) } async fn get_tile( diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs index 7d9689a01..5a73fd7ff 100755 --- a/martin/src/srv/tiles.rs +++ b/martin/src/srv/tiles.rs @@ -14,7 +14,7 @@ use martin_tile_utils::{ use serde::Deserialize; use crate::args::PreferredEncoding; -use crate::source::{Source, TileSources, UrlQuery}; +use crate::source::{Source, TileInfoSource, TileInfoSources, TileSources, UrlQuery}; use crate::srv::server::map_internal_error; use crate::srv::SrvConfig; use crate::utils::cache::get_or_insert_cached_value; @@ -62,7 +62,7 @@ async fn get_tile( } pub struct DynTileSource<'a> { - pub sources: Vec<&'a dyn Source>, + pub sources: TileInfoSources, pub info: TileInfo, pub query_str: Option<&'a str>, pub query_obj: Option, diff --git a/martin/src/srv/tiles_info.rs b/martin/src/srv/tiles_info.rs index a93313808..1e6b6305b 100755 --- a/martin/src/srv/tiles_info.rs +++ b/martin/src/srv/tiles_info.rs @@ -8,7 +8,7 @@ use itertools::Itertools as _; use serde::Deserialize; use tilejson::{tilejson, TileJSON}; -use crate::source::{Source, TileSources}; +use crate::source::{Source, TileInfoSource, TileSources}; use crate::srv::SrvConfig; #[derive(Deserialize)] @@ -62,7 +62,7 @@ async fn get_source_info( } #[must_use] -pub fn merge_tilejson(sources: &[&dyn Source], tiles_url: String) -> TileJSON { +pub fn merge_tilejson(sources: &[TileInfoSource], tiles_url: String) -> TileJSON { if sources.len() == 1 { let mut tj = sources[0].get_tilejson().clone(); tj.tiles = vec![tiles_url]; @@ -183,7 +183,7 @@ pub mod tests { }, data: Vec::default(), }; - let tj = merge_tilejson(&[&src1], url.clone()); + let tj = merge_tilejson(&[Box::new(src1.clone())], url.clone()); assert_eq!( TileJSON { tiles: vec![url.clone()], @@ -210,7 +210,7 @@ pub mod tests { data: Vec::default(), }; - let tj = merge_tilejson(&[&src1, &src2], url.clone()); + let tj = merge_tilejson(&[Box::new(src1.clone()), Box::new(src2)], url.clone()); assert_eq!(tj.tiles, vec![url]); assert_eq!(tj.name, Some("layer1,layer2".to_string())); assert_eq!(tj.minzoom, Some(5)); diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs index 02116664b..151bca711 100644 --- a/martin/tests/utils/pg_utils.rs +++ b/martin/tests/utils/pg_utils.rs @@ -44,7 +44,7 @@ pub fn table<'a>(mock: &'a MockSource, name: &str) -> &'a martin::pg::TableInfo #[allow(dead_code)] #[must_use] -pub fn source<'a>(mock: &'a MockSource, name: &str) -> &'a dyn Source { +pub fn source(mock: &MockSource, name: &str) -> Box { let (sources, _) = mock; sources.tiles.get_source(name).unwrap() }