feat!: DashMap based catalog

Cargo.toml

@@ -40,6 +40,7 @@ brotli = ">=5, <8"
 clap = { version = "4", features = ["derive"] }
 criterion = { version = "0.5", features = ["async_futures", "async_tokio", "html_reports"] }
 ctor = "0.3"
+dashmap = { version =  "6.1.0", features = ["serde", "inline", "rayon"] }
 deadpool-postgres = "0.14"
 enum-display = "0.1"
 env_logger = "0.11"

martin/Cargo.toml

@@ -75,6 +75,7 @@ actix-web.workspace = true
 async-trait.workspace = true
 bit-set = { workspace = true, optional = true }
 clap.workspace = true
+dashmap.workspace = true
 deadpool-postgres = { workspace = true, optional = true }
 enum-display.workspace = true
 env_logger.workspace = true

martin/src/bin/martin-cp.rs

@@ -14,8 +14,8 @@ use log::{debug, error, info, log_enabled};
 use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs};
 use martin::srv::{merge_tilejson, DynTileSource};
 use martin::{
-    append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileData,
-    TileRect,
+    append_rect, read_config, Config, MartinError, MartinResult, ServerState, TileData,
+    TileInfoSource, TileRect,
 };
 use martin_tile_utils::{bbox_to_xyz, TileCoord, TileInfo};
 use mbtiles::sqlx::SqliteConnection;
@@ -392,7 +392,7 @@ fn parse_encoding(encoding: &str) -> MartinCpResult<AcceptEncoding> {
 async fn init_schema(
     mbt: &Mbtiles,
     conn: &mut SqliteConnection,
-    sources: &[&dyn Source],
+    sources: &[TileInfoSource],
     tile_info: TileInfo,
     args: &CopyArgs,
 ) -> Result<MbtType, MartinError> {

martin/src/fonts/mod.rs

@@ -1,11 +1,10 @@
-use std::collections::hash_map::Entry;
-use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsStr;
 use std::fmt::Debug;
 use std::path::PathBuf;
 use std::sync::LazyLock;
 
 use bit_set::BitSet;
+use dashmap::{DashMap, Entry};
 use itertools::Itertools as _;
 use log::{debug, info, warn};
 use pbf_font_tools::freetype::{Face, Library};
@@ -103,11 +102,11 @@ fn get_available_codepoints(face: &mut Face) -> Option<GetGlyphInfo> {
 
 #[derive(Debug, Clone, Default)]
 pub struct FontSources {
-    fonts: HashMap<String, FontSource>,
+    fonts: DashMap<String, FontSource>,
     masks: Vec<BitSet>,
 }
 
-pub type FontCatalog = BTreeMap<String, CatalogFontEntry>;
+pub type FontCatalog = DashMap<String, CatalogFontEntry>;
 
 #[serde_with::skip_serializing_none]
 #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
@@ -125,7 +124,7 @@ impl FontSources {
             return Ok(Self::default());
         }
 
-        let mut fonts = HashMap::new();
+        let mut fonts = DashMap::new();
         let lib = Library::init()?;
 
         for path in config.iter() {
@@ -150,7 +149,7 @@ impl FontSources {
     pub fn get_catalog(&self) -> FontCatalog {
         self.fonts
             .iter()
-            .map(|(k, v)| (k.clone(), v.catalog_entry.clone()))
+            .map(|v| (v.key().clone(), v.catalog_entry.clone()))
             .sorted_by(|(a, _), (b, _)| a.cmp(b))
             .collect()
     }
@@ -242,7 +241,7 @@ pub struct FontSource {
 fn recurse_dirs(
     lib: &Library,
     path: PathBuf,
-    fonts: &mut HashMap<String, FontSource>,
+    fonts: &mut DashMap<String, FontSource>,
     is_top_level: bool,
 ) -> FontResult<()> {
     let start_count = fonts.len();
@@ -275,7 +274,7 @@ fn recurse_dirs(
 
 fn parse_font(
     lib: &Library,
-    fonts: &mut HashMap<String, FontSource>,
+    fonts: &mut DashMap<String, FontSource>,
     path: PathBuf,
 ) -> FontResult<()> {
     static RE_SPACES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\s|/|,)+").unwrap());

martin/src/lib.rs

@@ -5,7 +5,9 @@ mod config;
 pub use config::{read_config, Config, ServerState};
 
 mod source;
-pub use source::{CatalogSourceEntry, Source, Tile, TileData, TileSources, UrlQuery};
+pub use source::{
+    CatalogSourceEntry, Source, Tile, TileData, TileInfoSource, TileSources, UrlQuery,
+};
 
 mod utils;
 pub use utils::{

martin/src/source.rs

@@ -1,8 +1,9 @@
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::fmt::Debug;
 
 use actix_web::error::ErrorNotFound;
 use async_trait::async_trait;
+use dashmap::DashMap;
 use log::debug;
 use martin_tile_utils::{TileCoord, TileInfo};
 use serde::{Deserialize, Serialize};
@@ -18,8 +19,8 @@ pub type TileInfoSource = Box<dyn Source>;
 pub type TileInfoSources = Vec<TileInfoSource>;
 
 #[derive(Default, Clone)]
-pub struct TileSources(HashMap<String, TileInfoSource>);
-pub type TileCatalog = BTreeMap<String, CatalogSourceEntry>;
+pub struct TileSources(DashMap<String, TileInfoSource>);
+pub type TileCatalog = DashMap<String, CatalogSourceEntry>;
 
 impl TileSources {
     #[must_use]
@@ -37,16 +38,17 @@ impl TileSources {
     pub fn get_catalog(&self) -> TileCatalog {
         self.0
             .iter()
-            .map(|(id, src)| (id.to_string(), src.get_catalog_entry()))
+            .map(|v| (v.key().to_string(), v.get_catalog_entry()))
             .collect()
     }
 
-    pub fn get_source(&self, id: &str) -> actix_web::Result<&dyn Source> {
+    pub fn get_source(&self, id: &str) -> actix_web::Result<TileInfoSource> {
         Ok(self
             .0
             .get(id)
             .ok_or_else(|| ErrorNotFound(format!("Source {id} does not exist")))?
-            .as_ref())
+            .value()
+            .clone())
     }
 
     /// Get a list of sources, and the tile info for the merged sources.
@@ -56,7 +58,7 @@ impl TileSources {
         &self,
         source_ids: &str,
         zoom: Option<u8>,
-    ) -> actix_web::Result<(Vec<&dyn Source>, bool, TileInfo)> {
+    ) -> actix_web::Result<(Vec<TileInfoSource>, bool, TileInfo)> {
         let mut sources = Vec::new();
         let mut info: Option<TileInfo> = None;
         let mut use_url_query = false;
@@ -78,7 +80,7 @@ impl TileSources {
 
             // TODO: Use chained-if-let once available
             if match zoom {
-                Some(zoom) if Self::check_zoom(src, id, zoom) => true,
+                Some(zoom) if Self::check_zoom(&*src, id, zoom) => true,
                 None => true,
                 _ => false,
             } {

martin/src/sprites/mod.rs

@@ -1,15 +1,14 @@
-use std::collections::hash_map::Entry;
-use std::collections::{BTreeMap, HashMap};
-use std::fmt::Debug;
-use std::path::PathBuf;
-
+use dashmap::{DashMap, Entry};
 use futures::future::try_join_all;
 use log::{info, warn};
 use serde::{Deserialize, Serialize};
 use spreet::resvg::usvg::{Error as ResvgError, Options, Tree, TreeParsing};
 use spreet::{
     get_svg_input_paths, sprite_name, SpreetError, Sprite, Spritesheet, SpritesheetBuilder,
 };
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::path::PathBuf;
 use tokio::io::AsyncReadExt;
 
 use self::SpriteError::{SpriteInstError, SpriteParsingError, SpriteProcessingError};
@@ -56,7 +55,7 @@ pub struct CatalogSpriteEntry {
     pub images: Vec<String>,
 }
 
-pub type SpriteCatalog = BTreeMap<String, CatalogSpriteEntry>;
+pub type SpriteCatalog = DashMap<String, CatalogSpriteEntry>;
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct SpriteConfig {
@@ -71,7 +70,7 @@ impl ConfigExtras for SpriteConfig {
 }
 
 #[derive(Debug, Clone, Default)]
-pub struct SpriteSources(HashMap<String, SpriteSource>);
+pub struct SpriteSources(DashMap<String, SpriteSource>);
 
 impl SpriteSources {
     pub fn resolve(config: &mut FileConfigEnum<SpriteConfig>) -> FileResult<Self> {
@@ -109,8 +108,8 @@ impl SpriteSources {
 
     pub fn get_catalog(&self) -> SpriteResult<SpriteCatalog> {
         // TODO: all sprite generation should be pre-cached
-        let mut entries = SpriteCatalog::new();
-        for (id, source) in &self.0 {
+        let entries = SpriteCatalog::new();
+        for source in &self.0 {
             let paths = get_svg_input_paths(&source.path, true)
                 .map_err(|e| SpriteProcessingError(e, source.path.clone()))?;
             let mut images = Vec::with_capacity(paths.len());
@@ -121,7 +120,7 @@ impl SpriteSources {
                 );
             }
             images.sort();
-            entries.insert(id.clone(), CatalogSpriteEntry { images });
+            entries.insert(source.key().clone(), CatalogSpriteEntry { images });
         }
         Ok(entries)
     }
@@ -141,7 +140,7 @@ impl SpriteSources {
                     v.insert(SpriteSource { path });
                 }
             }
-        };
+        }
     }
 
     /// Given a list of IDs in a format "id1,id2,id3", return a spritesheet with them all.
@@ -155,14 +154,17 @@ impl SpriteSources {
 
         let sprite_ids = ids
             .split(',')
-            .map(|id| {
-                self.0
-                    .get(id)
-                    .ok_or_else(|| SpriteError::SpriteNotFound(id.to_string()))
-            })
+            .map(|id| self.get(id))
             .collect::<SpriteResult<Vec<_>>>()?;
 
-        get_spritesheet(sprite_ids.into_iter(), dpi, as_sdf).await
+        get_spritesheet(sprite_ids.iter(), dpi, as_sdf).await
+    }
+
+    fn get(&self, id: &str) -> SpriteResult<SpriteSource> {
+        match self.0.get(id) {
+            Some(v) => Ok(v.clone()),
+            None => Err(SpriteError::SpriteNotFound(id.to_string())),
+        }
     }
 }
 
@@ -247,14 +249,28 @@ mod tests {
         //.sdf => generate sdf from png, add sdf == true
         //- => does not generate sdf, omits sdf == true
         for extension in ["_sdf", ""] {
-            test_src(sprites.values(), 1, "all_1", extension).await;
-            test_src(sprites.values(), 2, "all_2", extension).await;
-
-            test_src(sprites.get("src1").into_iter(), 1, "src1_1", extension).await;
-            test_src(sprites.get("src1").into_iter(), 2, "src1_2", extension).await;
-
-            test_src(sprites.get("src2").into_iter(), 1, "src2_1", extension).await;
-            test_src(sprites.get("src2").into_iter(), 2, "src2_2", extension).await;
+            let paths = sprites
+                .iter()
+                .map(|v| v.value().clone())
+                .collect::<Vec<_>>();
+            test_src(paths.iter(), 1, "all_1", extension).await;
+            test_src(paths.iter(), 2, "all_2", extension).await;
+
+            let src1_path = sprites
+                .get("src1")
+                .into_iter()
+                .map(|v| v.value().clone())
+                .collect::<Vec<_>>();
+            test_src(src1_path.iter(), 1, "src1_1", extension).await;
+            test_src(src1_path.iter(), 2, "src1_2", extension).await;
+
+            let src2_path = sprites
+                .get("src2")
+                .into_iter()
+                .map(|v| v.value().clone())
+                .collect::<Vec<_>>();
+            test_src(src2_path.iter(), 1, "src2_1", extension).await;
+            test_src(src2_path.iter(), 2, "src2_2", extension).await;
         }
     }
 

martin/src/srv/server.rs

@@ -40,7 +40,7 @@ pub const RESERVED_KEYWORDS: &[&str] = &[
     "reload", "sprite", "status",
 ];
 
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct Catalog {
     pub tiles: TileCatalog,
     #[cfg(feature = "sprites")]
@@ -226,7 +226,7 @@ pub mod tests {
         }
 
         fn clone_source(&self) -> TileInfoSource {
-            unimplemented!()
+            Box::new(self.clone())
         }
 
         async fn get_tile(

martin/src/srv/tiles.rs

@@ -14,7 +14,7 @@ use martin_tile_utils::{
 use serde::Deserialize;
 
 use crate::args::PreferredEncoding;
-use crate::source::{Source, TileSources, UrlQuery};
+use crate::source::{TileInfoSources, TileSources, UrlQuery};
 use crate::srv::server::map_internal_error;
 use crate::srv::SrvConfig;
 use crate::utils::cache::get_or_insert_cached_value;
@@ -62,7 +62,7 @@ async fn get_tile(
 }
 
 pub struct DynTileSource<'a> {
-    pub sources: Vec<&'a dyn Source>,
+    pub sources: TileInfoSources,
     pub info: TileInfo,
     pub query_str: Option<&'a str>,
     pub query_obj: Option<UrlQuery>,