-
Notifications
You must be signed in to change notification settings - Fork 350
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
move tantivy val to json to own module remove duplicated conversion logic move field_presence to own module
- Loading branch information
Showing
5 changed files
with
545 additions
and
482 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
quickwit/quickwit-doc-mapper/src/doc_mapper/field_presence.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| use fnv::FnvHashSet; | ||
| use quickwit_common::PathHasher; | ||
| use tantivy::schema::document::{ReferenceValue, ReferenceValueLeaf}; | ||
| use tantivy::schema::{FieldType, Schema, Value}; | ||
| use tantivy::Document; | ||
|
|
||
| /// Populates the field presence for a document. | ||
| /// | ||
| /// The field presence is a set of hashes that represent the fields that are present in the | ||
| /// document. Each hash is computed from the field path. | ||
| /// | ||
| /// It is only added if the field is indexed and not fast. | ||
| pub(crate) fn populate_field_presence<D: Document>( | ||
| document: &D, | ||
| schema: &Schema, | ||
| ) -> FnvHashSet<u64> { | ||
| let mut field_presence_hashes: FnvHashSet<u64> = | ||
| FnvHashSet::with_capacity_and_hasher(schema.num_fields(), Default::default()); | ||
| for (field, value) in document.iter_fields_and_values() { | ||
| let field_entry = schema.get_field_entry(field); | ||
| if !field_entry.is_indexed() || field_entry.is_fast() { | ||
| // We are using an tantivy's ExistsQuery for fast fields. | ||
| continue; | ||
| } | ||
| let mut path_hasher: PathHasher = PathHasher::default(); | ||
| path_hasher.append(&field.field_id().to_le_bytes()[..]); | ||
| if let Some(json_obj) = value.as_object() { | ||
| let is_expand_dots_enabled: bool = | ||
| if let FieldType::JsonObject(json_options) = field_entry.field_type() { | ||
| json_options.is_expand_dots_enabled() | ||
| } else { | ||
| false | ||
| }; | ||
| populate_field_presence_for_json_obj( | ||
| json_obj, | ||
| path_hasher, | ||
| is_expand_dots_enabled, | ||
| &mut field_presence_hashes, | ||
| ); | ||
| } else { | ||
| field_presence_hashes.insert(path_hasher.finish()); | ||
| } | ||
| } | ||
| field_presence_hashes | ||
| } | ||
|
|
||
| #[inline] | ||
| fn populate_field_presence_for_json_value<'a>( | ||
| json_value: impl Value<'a>, | ||
| path_hasher: &PathHasher, | ||
| is_expand_dots_enabled: bool, | ||
| output: &mut FnvHashSet<u64>, | ||
| ) { | ||
| match json_value.as_value() { | ||
| ReferenceValue::Leaf(ReferenceValueLeaf::Null) => {} | ||
| ReferenceValue::Leaf(_) => { | ||
| output.insert(path_hasher.finish()); | ||
| } | ||
| ReferenceValue::Array(items) => { | ||
| for item in items { | ||
| populate_field_presence_for_json_value( | ||
| item, | ||
| path_hasher, | ||
| is_expand_dots_enabled, | ||
| output, | ||
| ); | ||
| } | ||
| } | ||
| ReferenceValue::Object(json_obj) => { | ||
| populate_field_presence_for_json_obj( | ||
| json_obj, | ||
| path_hasher.clone(), | ||
| is_expand_dots_enabled, | ||
| output, | ||
| ); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fn populate_field_presence_for_json_obj<'a, Iter: Iterator<Item = (&'a str, impl Value<'a>)>>( | ||
| json_obj: Iter, | ||
| path_hasher: PathHasher, | ||
| is_expand_dots_enabled: bool, | ||
| output: &mut FnvHashSet<u64>, | ||
| ) { | ||
| for (field_key, field_value) in json_obj { | ||
| let mut child_path_hasher = path_hasher.clone(); | ||
| if is_expand_dots_enabled { | ||
| for segment in field_key.split('.') { | ||
| child_path_hasher.append(segment.as_bytes()); | ||
| } | ||
| } else { | ||
| child_path_hasher.append(field_key.as_bytes()); | ||
| }; | ||
| populate_field_presence_for_json_value( | ||
| field_value, | ||
| &child_path_hasher, | ||
| is_expand_dots_enabled, | ||
| output, | ||
| ); | ||
| } | ||
| } |
Oops, something went wrong.