From 835b7d83cfa115b1a2017bcc6ad6bd7d0392cd69 Mon Sep 17 00:00:00 2001 From: Abdulla Abdurakhmanov Date: Fri, 6 Oct 2023 17:57:07 +0200 Subject: [PATCH] Query caching initial support --- examples/caching_memory_collections.rs | 6 + examples/caching_persistent_collections.rs | 6 + src/cache/backends/memory_backend.rs | 51 ++- src/cache/cache_filter_engine.rs | 461 +++++++++++++++++++++ src/cache/cache_query_engine.rs | 33 ++ src/cache/mod.rs | 3 + src/firestore_document_functions.rs | 36 ++ src/lib.rs | 3 + 8 files changed, 588 insertions(+), 11 deletions(-) create mode 100644 src/cache/cache_filter_engine.rs create mode 100644 src/cache/cache_query_engine.rs create mode 100644 src/firestore_document_functions.rs diff --git a/examples/caching_memory_collections.rs b/examples/caching_memory_collections.rs index 621edee..1126387 100644 --- a/examples/caching_memory_collections.rs +++ b/examples/caching_memory_collections.rs @@ -164,6 +164,12 @@ async fn main() -> Result<(), Box> { .fluent() .select() .from(TEST_COLLECTION_NAME) + .filter(|q| { + q.for_all( + q.field(path!(MyTestStructure::some_num)) + .greater_than_or_equal(2), + ) + }) .obj::() .stream_query_with_errors() .await?; diff --git a/examples/caching_persistent_collections.rs b/examples/caching_persistent_collections.rs index ea7e232..4650a6e 100644 --- a/examples/caching_persistent_collections.rs +++ b/examples/caching_persistent_collections.rs @@ -164,6 +164,12 @@ async fn main() -> Result<(), Box> { .fluent() .select() .from(TEST_COLLECTION_NAME) + .filter(|q| { + q.for_all( + q.field(path!(MyTestStructure::some_num)) + .greater_than_or_equal(250), + ) + }) .obj::() .stream_query_with_errors() .await?; diff --git a/src/cache/backends/memory_backend.rs b/src/cache/backends/memory_backend.rs index bad9814..33fabf7 100644 --- a/src/cache/backends/memory_backend.rs +++ b/src/cache/backends/memory_backend.rs @@ -5,7 +5,9 @@ use chrono::Utc; use futures::stream::BoxStream; use moka::future::{Cache, CacheBuilder}; +use crate::cache::cache_query_engine::FirestoreCacheQueryEngine; use futures::TryStreamExt; +use futures::{future, StreamExt}; use std::collections::HashMap; use tracing::*; @@ -99,6 +101,40 @@ impl FirestoreMemoryCacheBackend { } Ok(()) } + + async fn query_cached_docs( + &self, + collection_path: &str, + query_engine: FirestoreCacheQueryEngine, + ) -> FirestoreResult>> { + match self.collection_caches.get(collection_path) { + Some(mem_cache) => Ok(Box::pin( + futures::stream::unfold( + (query_engine, mem_cache.iter()), + |(query_engine, mut iter)| async move { + match iter.next() { + Some((_, doc)) => { + if query_engine.matches_doc(&doc) { + Some((Ok(Some(doc)), (query_engine, iter))) + } else { + Some((Ok(None), (query_engine, iter))) + } + } + None => None, + } + }, + ) + .filter_map(|doc_res| { + future::ready(match doc_res { + Ok(Some(doc)) => Some(Ok(doc)), + Ok(None) => None, + Err(err) => Some(Err(err)), + }) + }), + )), + None => Ok(Box::pin(futures::stream::empty())), + } + } } #[async_trait] @@ -221,18 +257,11 @@ impl FirestoreCacheDocsByPathSupport for FirestoreMemoryCacheBackend { collection_path: &str, query: &FirestoreQueryParams, ) -> FirestoreResult>>> { - // For now only basic/simple query all supported - if query.all_descendants.iter().all(|x| !*x) - && query.order_by.is_none() - && query.filter.is_none() - && query.start_at.is_none() - && query.end_at.is_none() - && query.offset.is_none() - && query.limit.is_none() - && query.return_only_fields.is_none() - { + let simple_query_engine = FirestoreCacheQueryEngine::new(query); + if simple_query_engine.params_supported() { Ok(FirestoreCachedValue::UseCached( - self.list_all_docs(collection_path).await?, + self.query_cached_docs(collection_path, simple_query_engine) + .await?, )) } else { Ok(FirestoreCachedValue::SkipCache) diff --git a/src/cache/cache_filter_engine.rs b/src/cache/cache_filter_engine.rs new file mode 100644 index 0000000..a5fa2a7 --- /dev/null +++ b/src/cache/cache_filter_engine.rs @@ -0,0 +1,461 @@ +use crate::FirestoreQueryFilter; +use crate::*; + +pub struct FirestoreCacheFilterEngine<'a> { + filter: &'a FirestoreQueryFilter, +} + +impl<'a> FirestoreCacheFilterEngine<'a> { + pub fn new(filter: &'a FirestoreQueryFilter) -> Self { + Self { filter } + } + + pub fn matches_doc(&'a self, doc: &FirestoreDocument) -> bool { + Self::matches_doc_filter(doc, &self.filter) + } + + pub fn matches_doc_filter(doc: &FirestoreDocument, filter: &FirestoreQueryFilter) -> bool { + match filter { + FirestoreQueryFilter::Composite(composite_filter) => match composite_filter.operator { + FirestoreQueryFilterCompositeOperator::And => composite_filter + .for_all_filters + .iter() + .all(|filter| Self::matches_doc_filter(doc, filter)), + + FirestoreQueryFilterCompositeOperator::Or => composite_filter + .for_all_filters + .iter() + .any(|filter| Self::matches_doc_filter(doc, filter)), + }, + FirestoreQueryFilter::Unary(unary_filter) => { + Self::matches_doc_filter_unary(doc, unary_filter) + } + FirestoreQueryFilter::Compare(compare_filter) => { + Self::matches_doc_filter_compare(doc, compare_filter) + } + } + } + + pub fn matches_doc_filter_unary( + doc: &FirestoreDocument, + filter: &FirestoreQueryFilterUnary, + ) -> bool { + match filter { + FirestoreQueryFilterUnary::IsNan(field_path) => { + firestore_doc_get_field_by_path(doc, field_path) + .map(|field_value| match field_value { + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue( + double_value, + ) => double_value.is_nan(), + _ => false, + }) + .unwrap_or(false) + } + FirestoreQueryFilterUnary::IsNotNan(field_path) => { + firestore_doc_get_field_by_path(doc, field_path) + .map(|field_value| match field_value { + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue( + double_value, + ) => !double_value.is_nan(), + _ => true, + }) + .unwrap_or(true) + } + FirestoreQueryFilterUnary::IsNull(field_path) => { + firestore_doc_get_field_by_path(doc, field_path) + .map(|field_value| match field_value { + gcloud_sdk::google::firestore::v1::value::ValueType::NullValue(_) => true, + _ => false, + }) + .unwrap_or(true) + } + FirestoreQueryFilterUnary::IsNotNull(field_path) => { + firestore_doc_get_field_by_path(doc, field_path) + .map(|field_value| match field_value { + gcloud_sdk::google::firestore::v1::value::ValueType::NullValue(_) => false, + _ => true, + }) + .unwrap_or(false) + } + } + } + + pub fn matches_doc_filter_compare( + doc: &FirestoreDocument, + filter: &Option, + ) -> bool { + match filter { + Some(FirestoreQueryFilterCompare::Equal(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values(CompareOp::Equal, field_value, compare_with_value) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::LessThan(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values(CompareOp::LessThan, field_value, compare_with_value) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::LessThanOrEqual(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values( + CompareOp::LessThanOrEqual, + field_value, + compare_with_value, + ) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::GreaterThan(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values( + CompareOp::GreaterThan, + field_value, + compare_with_value, + ) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::GreaterThanOrEqual(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values( + CompareOp::GreaterThanOrEqual, + field_value, + compare_with_value, + ) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::NotEqual(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values(CompareOp::NotEqual, field_value, compare_with_value) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::ArrayContains(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values( + CompareOp::ArrayContains, + field_value, + compare_with_value, + ) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::In(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values(CompareOp::In, field_value, compare_with_value) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::ArrayContainsAny(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values( + CompareOp::ArrayContainsAny, + field_value, + compare_with_value, + ) + }) + }) + .unwrap_or(false) + } + Some(FirestoreQueryFilterCompare::NotIn(field_path, compare_with)) => { + firestore_doc_get_field_by_path(doc, field_path) + .and_then(|field_value| { + compare_with + .value + .value_type + .as_ref() + .map(|compare_with_value| { + compare_values(CompareOp::NotIn, field_value, compare_with_value) + }) + }) + .unwrap_or(false) + } + None => true, + } + } +} + +enum CompareOp { + Equal, + NotEqual, + LessThan, + LessThanOrEqual, + GreaterThan, + GreaterThanOrEqual, + ArrayContains, + ArrayContainsAny, + In, + NotIn, +} + +fn compare_values( + op: CompareOp, + a: &gcloud_sdk::google::firestore::v1::value::ValueType, + b: &gcloud_sdk::google::firestore::v1::value::ValueType, +) -> bool { + match (op, a, b) { + // handle BooleanValue + ( + CompareOp::Equal, + gcloud_sdk::google::firestore::v1::value::ValueType::BooleanValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::BooleanValue(b_val), + ) => a_val == b_val, + + ( + CompareOp::NotEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::BooleanValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::BooleanValue(b_val), + ) => a_val != b_val, + + // handle IntegerValue + ( + CompareOp::Equal, + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(b_val), + ) => a_val == b_val, + + ( + CompareOp::NotEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(b_val), + ) => a_val != b_val, + + ( + CompareOp::LessThan, + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(b_val), + ) => a_val < b_val, + + ( + CompareOp::LessThanOrEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(b_val), + ) => a_val <= b_val, + + ( + CompareOp::GreaterThan, + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(b_val), + ) => a_val > b_val, + + ( + CompareOp::GreaterThanOrEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::IntegerValue(b_val), + ) => a_val >= b_val, + + // For DoubleValue + ( + CompareOp::Equal, + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(b_val), + ) => a_val == b_val, + + ( + CompareOp::NotEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(b_val), + ) => a_val != b_val, + + ( + CompareOp::LessThan, + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(b_val), + ) => a_val < b_val, + + ( + CompareOp::LessThanOrEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(b_val), + ) => a_val <= b_val, + + ( + CompareOp::GreaterThan, + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(b_val), + ) => a_val > b_val, + + ( + CompareOp::GreaterThanOrEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::DoubleValue(b_val), + ) => a_val >= b_val, + + // For TimestampValue, assumes it's a numerical timestamp; if it's a string or date type, adjust accordingly + ( + CompareOp::Equal, + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(b_val), + ) => a_val == b_val, + + ( + CompareOp::NotEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(b_val), + ) => a_val != b_val, + + ( + CompareOp::LessThan, + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(b_val), + ) => a_val.seconds < b_val.seconds && a_val.nanos < b_val.nanos, + + ( + CompareOp::LessThanOrEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(b_val), + ) => a_val.seconds <= b_val.seconds && a_val.nanos <= b_val.nanos, + + ( + CompareOp::GreaterThan, + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(b_val), + ) => a_val.seconds > b_val.seconds && a_val.nanos > b_val.nanos, + + ( + CompareOp::GreaterThanOrEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::TimestampValue(b_val), + ) => a_val.seconds >= b_val.seconds && a_val.nanos >= b_val.nanos, + + // For StringType only Equal, NotEqual operations make sense in a general context + ( + CompareOp::Equal, + gcloud_sdk::google::firestore::v1::value::ValueType::StringValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::StringValue(b_val), + ) => a_val == b_val, + + ( + CompareOp::NotEqual, + gcloud_sdk::google::firestore::v1::value::ValueType::StringValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::StringValue(b_val), + ) => a_val != b_val, + + // Array Operation + ( + CompareOp::ArrayContains, + gcloud_sdk::google::firestore::v1::value::ValueType::ArrayValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::ArrayValue(b_val), + ) => a_val + .values + .iter() + .map(|v| &v.value_type) + .flatten() + .any(|a_val| { + b_val + .values + .iter() + .map(|v| &v.value_type) + .flatten() + .all(|b_val| compare_values(CompareOp::Equal, a_val, b_val)) + }), + + ( + CompareOp::ArrayContainsAny, + gcloud_sdk::google::firestore::v1::value::ValueType::ArrayValue(a_val), + gcloud_sdk::google::firestore::v1::value::ValueType::ArrayValue(b_val), + ) => a_val + .values + .iter() + .map(|v| &v.value_type) + .flatten() + .any(|a_val| { + b_val + .values + .iter() + .map(|v| &v.value_type) + .flatten() + .any(|b_val| compare_values(CompareOp::Equal, a_val, b_val)) + }), + + ( + CompareOp::In, + gcloud_sdk::google::firestore::v1::value::ValueType::ArrayValue(a_val), + b_val, + ) => a_val + .values + .iter() + .map(|v| &v.value_type) + .flatten() + .any(|a_val| compare_values(CompareOp::Equal, a_val, b_val)), + + ( + CompareOp::NotIn, + gcloud_sdk::google::firestore::v1::value::ValueType::ArrayValue(a_val), + b_val, + ) => a_val + .values + .iter() + .map(|v| &v.value_type) + .flatten() + .any(|a_val| !compare_values(CompareOp::Equal, a_val, b_val)), + + // Any other combinations result in false + _ => false, + } +} diff --git a/src/cache/cache_query_engine.rs b/src/cache/cache_query_engine.rs new file mode 100644 index 0000000..ecede2d --- /dev/null +++ b/src/cache/cache_query_engine.rs @@ -0,0 +1,33 @@ +use crate::cache::cache_filter_engine::FirestoreCacheFilterEngine; +use crate::*; + +pub struct FirestoreCacheQueryEngine { + query: FirestoreQueryParams, +} + +impl FirestoreCacheQueryEngine { + pub fn new(query: &FirestoreQueryParams) -> Self { + Self { + query: query.clone(), + } + } + + pub fn params_supported(&self) -> bool { + self.query.all_descendants.iter().all(|x| !*x) + && self.query.order_by.is_none() + && self.query.start_at.is_none() + && self.query.end_at.is_none() + && self.query.offset.is_none() + && self.query.limit.is_none() + && self.query.return_only_fields.is_none() + } + + pub fn matches_doc(&self, doc: &FirestoreDocument) -> bool { + if let Some(filter) = &self.query.filter { + let filter_engine = FirestoreCacheFilterEngine::new(filter); + filter_engine.matches_doc(doc) + } else { + true + } + } +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 748edff..0e036ac 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -15,6 +15,9 @@ use futures::stream::BoxStream; use futures::StreamExt; use tracing::*; +mod cache_filter_engine; +mod cache_query_engine; + pub struct FirestoreCache where B: FirestoreCacheBackend + Send + Sync + 'static, diff --git a/src/firestore_document_functions.rs b/src/firestore_document_functions.rs new file mode 100644 index 0000000..d1ee983 --- /dev/null +++ b/src/firestore_document_functions.rs @@ -0,0 +1,36 @@ +use crate::FirestoreDocument; +use std::collections::HashMap; + +pub fn firestore_doc_get_field_by_path<'d>( + doc: &'d FirestoreDocument, + field_path: &str, +) -> Option<&'d gcloud_sdk::google::firestore::v1::value::ValueType> { + let field_path: Vec = field_path + .split(".") + .into_iter() + .map(|s| s.to_string().replace("`", "")) + .collect(); + firestore_doc_get_field_by_path_arr(&doc.fields, &field_path) +} + +fn firestore_doc_get_field_by_path_arr<'d>( + fields: &'d HashMap, + field_path_arr: &[String], +) -> Option<&'d gcloud_sdk::google::firestore::v1::value::ValueType> { + field_path_arr.first().and_then(|field_name| { + fields.get(field_name).and_then(|field_value| { + if field_path_arr.len() == 1 { + field_value.value_type.as_ref() + } else { + match field_value.value_type { + Some(gcloud_sdk::google::firestore::v1::value::ValueType::MapValue( + ref map_value, + )) => { + firestore_doc_get_field_by_path_arr(&map_value.fields, &field_path_arr[1..]) + } + _ => None, + } + } + }) + }) +} diff --git a/src/lib.rs b/src/lib.rs index be4fd74..70f0280 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -144,6 +144,9 @@ pub type FirestoreResult = std::result::Result; pub type FirestoreDocument = gcloud_sdk::google::firestore::v1::Document; +mod firestore_document_functions; +pub use firestore_document_functions::*; + mod fluent_api; pub use fluent_api::*;