Merge branch 'main' into trinity/update-tantivy

quickwit-oss · Jun 10, 2024 · 9a7b27b · 9a7b27b
2 parents 635793d + 55a7123
commit 9a7b27b
Show file tree

Hide file tree

Showing 6 changed files with 280 additions and 78 deletions.
diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md
@@ -676,8 +676,7 @@ This section describes search settings for a given index.
 
 | Variable      | Description   | Default value |
 | ------------- | ------------- | ------------- |
-| `default_search_fields`      | Default list of fields that will be used for search. The field names in this list may be declared
-explicitly in the schema, or may refer to a field captured by the dynamic mode.   | `None` |
+| `default_search_fields` | Default list of fields that will be used for search. The field names in this list may be declared explicitly in the schema, or may refer to a field captured by the dynamic mode. | `None` |
 
 ## Retention policy
 

diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/date_time_type.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/date_time_type.rs
@@ -18,7 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 use indexmap::IndexSet;
-use quickwit_datetime::{DateTimeInputFormat, DateTimeOutputFormat};
+use quickwit_datetime::{DateTimeInputFormat, DateTimeOutputFormat, TantivyDateTime};
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_json::Value as JsonValue;
 use tantivy::schema::{DateTimePrecision, OwnedValue as TantivyValue};
@@ -98,6 +98,29 @@ impl QuickwitDateTimeOptions {
         };
         Ok(TantivyValue::Date(date_time))
     }
+
+    pub(crate) fn reparse_tantivy_value(
+        &self,
+        tantivy_value: &TantivyValue,
+    ) -> Option<TantivyDateTime> {
+        match tantivy_value {
+            TantivyValue::Date(date) => Some(*date),
+            TantivyValue::Str(date_time_str) => {
+                quickwit_datetime::parse_date_time_str(date_time_str, &self.input_formats.0).ok()
+            }
+            TantivyValue::U64(timestamp_u64) => {
+                let timestamp_i64 = (*timestamp_u64).try_into().ok()?;
+                quickwit_datetime::parse_timestamp_int(timestamp_i64, &self.input_formats.0).ok()
+            }
+            TantivyValue::I64(timestamp_i64) => {
+                quickwit_datetime::parse_timestamp_int(*timestamp_i64, &self.input_formats.0).ok()
+            }
+            TantivyValue::F64(timestamp_f64) => {
+                quickwit_datetime::parse_timestamp_float(*timestamp_f64, &self.input_formats.0).ok()
+            }
+            _ => None,
+        }
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]

diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs
@@ -675,7 +675,6 @@ mod tests {
 
     use super::DefaultDocMapper;
     use crate::default_doc_mapper::field_mapping_entry::DEFAULT_TOKENIZER_NAME;
-    use crate::default_doc_mapper::mapping_tree::value_to_pretokenized;
     use crate::{
         DefaultDocMapperBuilder, DocMapper, DocParsingError, DOCUMENT_SIZE_FIELD_NAME,
         DYNAMIC_FIELD_NAME, FIELD_PRESENCE_FIELD_NAME, SOURCE_FIELD_NAME,
@@ -1762,7 +1761,7 @@ mod tests {
             }"#,
             "concat",
             r#"{"some_int": 25}"#,
-            vec![value_to_pretokenized(25).into()],
+            vec![25_u64.into()],
         );
         test_doc_from_json_test_aux(
             r#"{
@@ -1777,7 +1776,7 @@ mod tests {
             }"#,
             "concat",
             r#"{"some_int": 25}"#,
-            vec![value_to_pretokenized(25).into()],
+            vec![25_u64.into()],
         );
     }
 
@@ -1800,7 +1799,7 @@ mod tests {
             }"#,
             "concat",
             r#"{"some_bool": false}"#,
-            vec![value_to_pretokenized(false).into()],
+            vec![false.into()],
         );
         test_doc_from_json_test_aux(
             r#"{
@@ -1815,7 +1814,7 @@ mod tests {
             }"#,
             "concat",
             r#"{"some_bool": true}"#,
-            vec![value_to_pretokenized(true).into()],
+            vec![true.into()],
         );
     }
 

diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs
@@ -702,9 +702,9 @@ impl Default for QuickwitConcatenateOptions {
     }
 }
 
-impl From<QuickwitConcatenateOptions> for TextOptions {
+impl From<QuickwitConcatenateOptions> for JsonObjectOptions {
     fn from(quickwit_text_options: QuickwitConcatenateOptions) -> Self {
-        let mut text_options = TextOptions::default();
+        let mut text_options = JsonObjectOptions::default();
         let text_field_indexing = TextFieldIndexing::default()
             .set_index_option(quickwit_text_options.indexing_options.record)
             .set_fieldnorms(quickwit_text_options.indexing_options.fieldnorms)