From a415f3094b0df91e0cc93ad320971ca02290100c Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Tue, 9 Jul 2024 15:57:21 +0100 Subject: [PATCH 1/4] feat(es): respect format parameter in range query --- .../src/elastic_query_dsl/range_query.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs index 88b9df05c11..98473cc1537 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs @@ -18,8 +18,10 @@ // along with this program. If not, see . use std::ops::Bound; +use std::str::FromStr; use serde::Deserialize; +use quickwit_datetime::{DateTimeInputFormat, parse_date_time_str, StrptimeParser}; use crate::elastic_query_dsl::one_field_map::OneFieldMap; use crate::elastic_query_dsl::ConvertableToQueryAst; @@ -40,7 +42,6 @@ pub struct RangeQueryParams { lte: Option, #[serde(default)] boost: Option, - // Currently NO-OP (see #5109) #[serde(default)] format: Option, } @@ -56,8 +57,20 @@ impl ConvertableToQueryAst for RangeQuery { lt, lte, boost, - format: _, + format, } = self.value; + let (gt, gte, lt, lte) = + if let Some(JsonLiteral::String(fmt)) = format { + ( + gt.map(|v| parse_and_convert(v, &fmt)).transpose()?, + gte.map(|v| parse_and_convert(v, &fmt)).transpose()?, + lt.map(|v| parse_and_convert(v, &fmt)).transpose()?, + lte.map(|v| parse_and_convert(v, &fmt)).transpose()?, + ) + } else { + (gt, gte, lt, lte) + }; + let range_query_ast = crate::query_ast::RangeQuery { field, lower_bound: match (gt, gte) { @@ -81,3 +94,15 @@ impl ConvertableToQueryAst for RangeQuery { Ok(ast.boost(boost)) } } + + +fn parse_and_convert(value: JsonLiteral, fmt: &str) -> anyhow::Result { + if let JsonLiteral::String(s) = value { + let date_format = DateTimeInputFormat::from_str(fmt).unwrap(); + let date_time = parse_date_time_str(&s, &[date_format]).unwrap(); + Ok(JsonLiteral::String(date_time.to_string())) // TODO no `to_string` method + } else { + dbg!(value.clone()); + Ok(value) + } +} From 6887b471ef9258ea7387db05dd495827a4bc4001 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 10 Jul 2024 22:07:35 +0100 Subject: [PATCH 2/4] chore: implement parsing logic --- .../src/elastic_query_dsl/range_query.rs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs index 98473cc1537..846d4a428cf 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs @@ -20,8 +20,8 @@ use std::ops::Bound; use std::str::FromStr; +use quickwit_datetime::StrptimeParser; use serde::Deserialize; -use quickwit_datetime::{DateTimeInputFormat, parse_date_time_str, StrptimeParser}; use crate::elastic_query_dsl::one_field_map::OneFieldMap; use crate::elastic_query_dsl::ConvertableToQueryAst; @@ -59,13 +59,15 @@ impl ConvertableToQueryAst for RangeQuery { boost, format, } = self.value; - let (gt, gte, lt, lte) = - if let Some(JsonLiteral::String(fmt)) = format { + let (gt, gte, lt, lte) = if let Some(JsonLiteral::String(fmt)) = format { + let parser = StrptimeParser::from_str(&fmt).map_err(|reason| { + anyhow::anyhow!("failed to create parser from : {}; reason: {}", fmt, reason) + })?; ( - gt.map(|v| parse_and_convert(v, &fmt)).transpose()?, - gte.map(|v| parse_and_convert(v, &fmt)).transpose()?, - lt.map(|v| parse_and_convert(v, &fmt)).transpose()?, - lte.map(|v| parse_and_convert(v, &fmt)).transpose()?, + gt.map(|v| parse_and_convert(v, &parser)).transpose()?, + gte.map(|v| parse_and_convert(v, &parser)).transpose()?, + lt.map(|v| parse_and_convert(v, &parser)).transpose()?, + lte.map(|v| parse_and_convert(v, &parser)).transpose()?, ) } else { (gt, gte, lt, lte) @@ -95,14 +97,13 @@ impl ConvertableToQueryAst for RangeQuery { } } - -fn parse_and_convert(value: JsonLiteral, fmt: &str) -> anyhow::Result { - if let JsonLiteral::String(s) = value { - let date_format = DateTimeInputFormat::from_str(fmt).unwrap(); - let date_time = parse_date_time_str(&s, &[date_format]).unwrap(); - Ok(JsonLiteral::String(date_time.to_string())) // TODO no `to_string` method +fn parse_and_convert(literal: JsonLiteral, parser: &StrptimeParser) -> anyhow::Result { + if let JsonLiteral::String(date_time_str) = literal { + let parsed_date_time = parser + .parse_date_time(&date_time_str) + .map_err(|reason| anyhow::anyhow!("Failed to parse date time: {}", reason))?; + Ok(JsonLiteral::String(parsed_date_time.to_string())) } else { - dbg!(value.clone()); - Ok(value) + Ok(literal) } } From 1f13d9db6ce5b94f09327d3c774e2a76b6905f1b Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Fri, 12 Jul 2024 11:29:16 +0100 Subject: [PATCH 3/4] chore(es): add a test --- .../src/elastic_query_dsl/range_query.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs index 846d4a428cf..ad51e3d5d52 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs @@ -107,3 +107,40 @@ fn parse_and_convert(literal: JsonLiteral, parser: &StrptimeParser) -> anyhow::R Ok(literal) } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use quickwit_datetime::StrptimeParser; + + use crate::elastic_query_dsl::range_query::parse_and_convert; + use crate::JsonLiteral; + + #[test] + fn test_parse_and_convert() { + let parser = StrptimeParser::from_str("%Y-%m-%d %H:%M:%S").unwrap(); + + // valid datetime + let input = JsonLiteral::String("2022-12-30 05:45:00".to_string()); + let result = parse_and_convert(input, &parser)?; + assert_eq!( + result, + JsonLiteral::String("2022-12-30 5:45:00.0 +00:00:00".to_string()) + ); + + // invalid datetime + let input = JsonLiteral::String("invalid datetime".to_string()); + let result = parse_and_convert(input, &parser); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Failed to parse date time")); + + // non_string(number) input + let input = JsonLiteral::Number(27.into()); + let result = parse_and_convert(input.clone(), &parser)?; + assert_eq!(result, input); + } +} From a674c9392b1a3796e14e812fc0f17ac0cd761341 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Fri, 12 Jul 2024 11:44:54 +0100 Subject: [PATCH 4/4] chore(es): small syntax fix --- quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs index ad51e3d5d52..273496bbdeb 100644 --- a/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs +++ b/quickwit/quickwit-query/src/elastic_query_dsl/range_query.rs @@ -118,7 +118,7 @@ mod tests { use crate::JsonLiteral; #[test] - fn test_parse_and_convert() { + fn test_parse_and_convert() -> anyhow::Result<()> { let parser = StrptimeParser::from_str("%Y-%m-%d %H:%M:%S").unwrap(); // valid datetime @@ -142,5 +142,7 @@ mod tests { let input = JsonLiteral::Number(27.into()); let result = parse_and_convert(input.clone(), &parser)?; assert_eq!(result, input); + + Ok(()) } }