From ba833e40c73589cdbce85c351f4406a100689b0e Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 9 Nov 2023 21:45:06 -0500 Subject: [PATCH] Add Moka cache support --- Cargo.lock | 167 +++++++++++++++++++++++++++++ Cargo.toml | 1 + martin/Cargo.toml | 1 + martin/src/file_config.rs | 8 +- martin/src/pmtiles/file_pmtiles.rs | 22 ++-- martin/src/pmtiles/http_pmtiles.rs | 63 ++++++++--- 6 files changed, 232 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9560f9c14..3cf0f2a64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -367,6 +367,15 @@ dependencies = [ "zstd-safe 7.0.0", ] +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener", +] + [[package]] name = "async-recursion" version = "1.0.5" @@ -519,6 +528,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "bytecount" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" + [[package]] name = "bytemuck" version = "1.14.0" @@ -546,12 +561,43 @@ dependencies = [ "bytes", ] +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + [[package]] name = "cargo-husky" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" +[[package]] +name = "cargo-platform" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cast" version = "0.3.0" @@ -1093,6 +1139,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -1907,6 +1962,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "martin" version = "0.11.2" @@ -1933,6 +1997,7 @@ dependencies = [ "log", "martin-tile-utils", "mbtiles", + "moka", "num_cpus", "pbf_font_tools", "pmtiles", @@ -2068,6 +2133,30 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "moka" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8017ec3548ffe7d4cef7ac0e12b044c01164a74c0f3119420faeaf13490ad8b" +dependencies = [ + "async-lock", + "async-trait", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "futures-util", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "multimap" version = "0.9.1" @@ -2637,6 +2726,33 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804" +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.33" @@ -2682,6 +2798,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "rayon" version = "1.8.0" @@ -3105,6 +3230,9 @@ name = "semver" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -3290,6 +3418,21 @@ dependencies = [ "num", ] +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.9" @@ -3714,6 +3857,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -4015,6 +4164,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "triomphe" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" + [[package]] name = "try-lock" version = "0.2.4" @@ -4039,6 +4194,15 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -4215,6 +4379,9 @@ name = "uuid" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "getrandom", +] [[package]] name = "varint-rs" diff --git a/Cargo.toml b/Cargo.toml index 1b460a1a0..b27b38b51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ json-patch = "1.2" log = "0.4" martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" } mbtiles = { path = "./mbtiles", version = "0.8.0" } +moka = { version = "0.12", features = ["future"] } num_cpus = "1" pbf_font_tools = { version = "2.5.0", features = ["freetype"] } pmtiles = { version = "0.5", features = ["http-async", "mmap-async-tokio", "tilejson"] } diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 51e351239..2c2060a08 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -80,6 +80,7 @@ json-patch.workspace = true log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true +moka.workspace = true num_cpus.workspace = true pbf_font_tools.workspace = true pmtiles.workspace = true diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 1528c752d..87b59e6de 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -40,7 +40,7 @@ pub enum FileError { AquireConnError(String), #[error(r#"PMTiles error {0} processing {1}"#)] - PmtError(pmtiles::error::Error, String), + PmtError(pmtiles::PmtError, String), } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -178,7 +178,7 @@ pub struct FileConfigSource { pub path: PathBuf, } -async fn dummy_resolver(_id: String, _url: Url) -> Result, FileError> { +async fn dummy_resolver(_id: String, _url: Url) -> FileResult> { unreachable!() } @@ -203,13 +203,13 @@ pub async fn resolve_files_urls( extension: &str, new_source: &mut impl FnMut(String, PathBuf) -> Fut1, new_url_source: &mut impl FnMut(String, Url) -> Fut2, -) -> Result +) -> MartinResult where Fut1: Future, FileError>>, Fut2: Future, FileError>>, { resolve_int(config, idr, extension, true, new_source, new_url_source) - .map_err(crate::Error::from) + .map_err(crate::MartinError::from) .await } diff --git a/martin/src/pmtiles/file_pmtiles.rs b/martin/src/pmtiles/file_pmtiles.rs index d27fbd01b..2eb7b0f35 100644 --- a/martin/src/pmtiles/file_pmtiles.rs +++ b/martin/src/pmtiles/file_pmtiles.rs @@ -11,10 +11,10 @@ use pmtiles::mmap::MmapBackend; use pmtiles::{Compression, TileType}; use tilejson::TileJSON; -use crate::file_config::FileError; use crate::file_config::FileError::{InvalidMetadata, IoError}; -use crate::source::{Source, Tile, UrlQuery}; -use crate::{Error, Xyz}; +use crate::file_config::FileResult; +use crate::source::{Source, UrlQuery}; +use crate::{MartinResult, TileCoord, TileData}; #[derive(Clone)] pub struct PmtFileSource { @@ -36,11 +36,11 @@ impl Debug for PmtFileSource { } impl PmtFileSource { - pub async fn new_box(id: String, path: PathBuf) -> Result, FileError> { + pub async fn new_box(id: String, path: PathBuf) -> FileResult> { Ok(Box::new(PmtFileSource::new(id, path).await?)) } - async fn new(id: String, path: PathBuf) -> Result { + async fn new(id: String, path: PathBuf) -> FileResult { let backend = MmapBackend::try_from(path.as_path()) .await .map_err(|e| { @@ -70,8 +70,8 @@ impl PmtFileSource { id: String, path: PathBuf, reader: AsyncPmTilesReader, - ) -> Result { - let hdr = &reader.header; + ) -> FileResult { + let hdr = &reader.get_header(); if hdr.tile_type != TileType::Mvt && hdr.tile_compression != Compression::None { return Err(InvalidMetadata( @@ -145,14 +145,18 @@ impl Source for PmtFileSource { Box::new(self.clone()) } - async fn get_tile(&self, xyz: &Xyz, _url_query: &Option) -> Result { + async fn get_tile( + &self, + xyz: &TileCoord, + _url_query: &Option, + ) -> MartinResult { // TODO: optimize to return Bytes if let Some(t) = self .pmtiles .get_tile(xyz.z, u64::from(xyz.x), u64::from(xyz.y)) .await { - Ok(t.data.to_vec()) + Ok(t.to_vec()) } else { trace!( "Couldn't find tile data in {}/{}/{} of {}", diff --git a/martin/src/pmtiles/http_pmtiles.rs b/martin/src/pmtiles/http_pmtiles.rs index 5b9e2e0e5..583501aca 100644 --- a/martin/src/pmtiles/http_pmtiles.rs +++ b/martin/src/pmtiles/http_pmtiles.rs @@ -4,23 +4,49 @@ use std::sync::Arc; use async_trait::async_trait; use log::{trace, warn}; use martin_tile_utils::{Encoding, Format, TileInfo}; +use moka::future::Cache; use pmtiles::async_reader::AsyncPmTilesReader; +use pmtiles::cache::{DirCacheResult, DirectoryCache}; use pmtiles::http::HttpBackend; -use pmtiles::{Compression, TileType}; +use pmtiles::{Compression, Directory, TileType}; use reqwest::Client; use tilejson::TileJSON; use url::Url; -use crate::file_config::FileError; use crate::file_config::FileError::InvalidMetadata; -use crate::source::{Source, Tile, UrlQuery}; -use crate::{Error, Xyz}; +use crate::file_config::{FileError, FileResult}; +use crate::source::{Source, UrlQuery}; +use crate::{MartinResult, TileCoord, TileData}; + +type PmtReader = AsyncPmTilesReader; + +struct PmtCache(Cache); + +impl PmtCache { + fn new() -> Self { + Self(Cache::new(10_000)) + } +} + +#[async_trait] +impl DirectoryCache for PmtCache { + async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { + match self.0.get(&offset).await { + Some(dir) => dir.find_tile_id(tile_id).into(), + None => DirCacheResult::NotCached, + } + } + + async fn insert_dir(&self, offset: usize, directory: Directory) { + self.0.insert(offset, directory).await; + } +} #[derive(Clone)] pub struct PmtHttpSource { id: String, url: Url, - pmtiles: Arc>, + pmtiles: Arc, tilejson: TileJSON, tile_info: TileInfo, } @@ -36,25 +62,24 @@ impl Debug for PmtHttpSource { } impl PmtHttpSource { - pub async fn new_url_box(id: String, url: Url) -> Result, FileError> { + pub async fn new_url_box(id: String, url: Url) -> FileResult> { let client = Client::new(); - Ok(Box::new(PmtHttpSource::new_url(client, id, url).await?)) + let cache = PmtCache::new(); + Ok(Box::new( + PmtHttpSource::new_url(client, cache, id, url).await?, + )) } - async fn new_url(client: Client, id: String, url: Url) -> Result { - let reader = AsyncPmTilesReader::new_with_url(client, url.clone()).await; + async fn new_url(client: Client, cache: PmtCache, id: String, url: Url) -> FileResult { + let reader = AsyncPmTilesReader::new_with_cached_url(cache, client, url.clone()).await; let reader = reader.map_err(|e| FileError::PmtError(e, url.to_string()))?; Self::new_int(id, url, reader).await } } impl PmtHttpSource { - async fn new_int( - id: String, - url: Url, - reader: AsyncPmTilesReader, - ) -> Result { - let hdr = &reader.header; + async fn new_int(id: String, url: Url, reader: PmtReader) -> FileResult { + let hdr = &reader.get_header(); if hdr.tile_type != TileType::Mvt && hdr.tile_compression != Compression::None { return Err(InvalidMetadata( @@ -125,14 +150,18 @@ impl Source for PmtHttpSource { Box::new(self.clone()) } - async fn get_tile(&self, xyz: &Xyz, _url_query: &Option) -> Result { + async fn get_tile( + &self, + xyz: &TileCoord, + _url_query: &Option, + ) -> MartinResult { // TODO: optimize to return Bytes if let Some(t) = self .pmtiles .get_tile(xyz.z, u64::from(xyz.x), u64::from(xyz.y)) .await { - Ok(t.data.to_vec()) + Ok(t.to_vec()) } else { trace!( "Couldn't find tile data in {}/{}/{} of {}",