update tantivy (#4891)

* update tantivy * fix term construction * Apply suggestions from code review Co-authored-by: Paul Masurel <[email protected]> --------- Co-authored-by: Paul Masurel <[email protected]>
quickwit-oss · Apr 24, 2024 · 8d419c8 · 8d419c8
1 parent 69ad540
commit 8d419c8
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 49 deletions.
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml
@@ -320,7 +320,7 @@ quickwit-serve = { path = "quickwit-serve" }
 quickwit-storage = { path = "quickwit-storage" }
 quickwit-telemetry = { path = "quickwit-telemetry" }
 
-tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "74940e9", default-features = false, features = [
+tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "ff40764", default-features = false, features = [
   "lz4-compression",
   "mmap",
   "quickwit",

diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs
@@ -500,7 +500,7 @@ fn populate_field_presence_for_json_value(
 }
 
 fn populate_field_presence_for_json_obj(
-    json_obj: &BTreeMap<String, TantivyValue>,
+    json_obj: &[(String, TantivyValue)],
     path_hasher: PathHasher,
     is_expand_dots_enabled: bool,
     output: &mut FnvHashSet<u64>,

diff --git a/quickwit/quickwit-query/src/query_ast/full_text_query.rs b/quickwit/quickwit-query/src/query_ast/full_text_query.rs
@@ -19,7 +19,6 @@
 
 use anyhow::Context;
 use serde::{Deserialize, Serialize};
-use tantivy::json_utils::JsonTermWriter;
 use tantivy::query::{
     PhrasePrefixQuery as TantivyPhrasePrefixQuery, PhraseQuery as TantivyPhraseQuery,
     TermQuery as TantivyTermQuery,
@@ -79,16 +78,11 @@ impl FullTextParams {
             self.text_analyzer(text_indexing_options, tokenizer_manager)?;
         let mut token_stream = text_analyzer.token_stream(text);
         let mut tokens = Vec::new();
-        let mut term = Term::with_capacity(100);
-        let mut json_term_writer = JsonTermWriter::from_field_and_json_path(
-            field,
-            json_path,
-            json_options.is_expand_dots_enabled(),
-            &mut term,
-        );
         token_stream.process(&mut |token| {
-            json_term_writer.set_str(&token.text);
-            tokens.push((token.position, json_term_writer.term().clone()));
+            let mut term =
+                Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());
+            term.append_type_and_str(&token.text);
+            tokens.push((token.position, term));
         });
         Ok(tokens)
     }

diff --git a/quickwit/quickwit-query/src/query_ast/utils.rs b/quickwit/quickwit-query/src/query_ast/utils.rs
@@ -17,7 +17,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-use tantivy::json_utils::{convert_to_fast_value_and_get_term, JsonTermWriter};
+use tantivy::json_utils::convert_to_fast_value_and_append_to_json_term;
 use tantivy::query::TermQuery as TantivyTermQuery;
 use tantivy::schema::{
     Field, FieldEntry, FieldType, IndexRecordOption, JsonObjectOptions, Schema as TantivySchema,
@@ -186,14 +186,8 @@ fn compute_tantivy_ast_query_for_json(
     tokenizer_manager: &TokenizerManager,
 ) -> Result<TantivyQueryAst, InvalidQuery> {
     let mut bool_query = TantivyBoolQuery::default();
-    let mut term = Term::with_capacity(100);
-    let mut json_term_writer = JsonTermWriter::from_field_and_json_path(
-        field,
-        json_path,
-        json_options.is_expand_dots_enabled(),
-        &mut term,
-    );
-    if let Some(term) = convert_to_fast_value_and_get_term(&mut json_term_writer, text) {
+    let term = Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());
+    if let Some(term) = convert_to_fast_value_and_append_to_json_term(term, text) {
         bool_query
             .should
             .push(TantivyTermQuery::new(term, IndexRecordOption::Basic).into());

diff --git a/quickwit/quickwit-query/src/query_ast/wildcard_query.rs b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs
@@ -19,7 +19,6 @@
 
 use anyhow::{anyhow, bail, Context};
 use serde::{Deserialize, Serialize};
-use tantivy::json_utils::JsonTermWriter;
 use tantivy::schema::{Field, FieldType, Schema as TantivySchema};
 use tantivy::Term;
 
@@ -163,17 +162,15 @@ impl WildcardQuery {
                     })?;
                 let mut token_stream = normalizer.token_stream(&prefix);
                 let mut tokens = Vec::new();
-                let mut term = Term::with_capacity(100);
-                let mut json_term_writer = JsonTermWriter::from_field_and_json_path(
-                    field,
-                    json_path,
-                    json_options.is_expand_dots_enabled(),
-                    &mut term,
-                );
 
                 token_stream.process(&mut |token| {
-                    json_term_writer.set_str(&token.text);
-                    tokens.push(json_term_writer.term().clone());
+                    let mut term = Term::from_field_json_path(
+                        field,
+                        json_path,
+                        json_options.is_expand_dots_enabled(),
+                    );
+                    term.append_type_and_str(&token.text);
+                    tokens.push(term);
                 });
                 let term = extract_unique_token(tokens)?;
                 Ok((field, term))