Skip to content

Commit

Permalink
move debug assert to term, rename method
Browse files Browse the repository at this point in the history
  • Loading branch information
@PSeitz
PSeitz committed Nov 13, 2023
1 parent b53c48b commit 44e4aeb
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 41 deletions.
48 changes: 21 additions & 27 deletions src/core/json_utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
term_buffer.append_type_and_fast_value(val);
term_buffer.append_type_and_fast_value_after_json_path(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::I64(val) => {
Expand All @@ -171,7 +171,7 @@ fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
term_buffer.append_type_and_fast_value(val);
term_buffer.append_type_and_fast_value_after_json_path(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::F64(val) => {
Expand All @@ -180,7 +180,7 @@ fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
term_buffer.append_type_and_fast_value(val);
term_buffer.append_type_and_fast_value_after_json_path(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::Bool(val) => {
Expand All @@ -189,7 +189,7 @@ fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
term_buffer.append_type_and_fast_value(val);
term_buffer.append_type_and_fast_value_after_json_path(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::Date(val) => {
Expand All @@ -198,7 +198,7 @@ fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
term_buffer.append_type_and_fast_value(val);
term_buffer.append_type_and_fast_value_after_json_path(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::PreTokStr(_) => {
Expand Down Expand Up @@ -248,35 +248,29 @@ fn index_json_value<'a, V: Value<'a>>(
/// Tries to infer a JSON type from a string and append it to the term.
///
/// The term must be json + JSON path.
pub(crate) fn convert_to_fast_value_and_append(mut term: Term, phrase: &str) -> Option<Term> {
assert_eq!(
term.value()
.as_json_value_bytes()
.expect("expecting a Term with a json type and json path")
.as_serialized()
.len(),
0,
"JSON value bytes should be empty"
);
pub(crate) fn convert_to_fast_value_and_append_to_json_term(
mut term: Term,
phrase: &str,
) -> Option<Term> {
if let Ok(dt) = OffsetDateTime::parse(phrase, &Rfc3339) {
let dt_utc = dt.to_offset(UtcOffset::UTC);
term.append_type_and_fast_value(DateTime::from_utc(dt_utc));
term.append_type_and_fast_value_after_json_path(DateTime::from_utc(dt_utc));
return Some(term);
}
if let Ok(i64_val) = str::parse::<i64>(phrase) {
term.append_type_and_fast_value(i64_val);
term.append_type_and_fast_value_after_json_path(i64_val);
return Some(term);
}
if let Ok(u64_val) = str::parse::<u64>(phrase) {
term.append_type_and_fast_value(u64_val);
term.append_type_and_fast_value_after_json_path(u64_val);
return Some(term);
}
if let Ok(f64_val) = str::parse::<f64>(phrase) {
term.append_type_and_fast_value(f64_val);
term.append_type_and_fast_value_after_json_path(f64_val);
return Some(term);
}
if let Ok(bool_val) = str::parse::<bool>(phrase) {
term.append_type_and_fast_value(bool_val);
term.append_type_and_fast_value_after_json_path(bool_val);
return Some(term);
}
None
Expand Down Expand Up @@ -364,7 +358,7 @@ mod tests {
let field = Field::from_field_id(1);

let mut term = term_from_json_paths(field, ["attributes", "color"].into_iter(), false);
term.append_type_and_str("red");
term.append_type_and_str_after_json_path("red");
assert_eq!(
format!("{:?}", term),
"Term(field=1, type=Json, path=attributes.color, type=Str, \"red\")"
Expand All @@ -375,7 +369,7 @@ mod tests {
["attributes", "dimensions", "width"].into_iter(),
false,
);
term.append_type_and_fast_value(400i64);
term.append_type_and_fast_value_after_json_path(400i64);
assert_eq!(
format!("{:?}", term),
"Term(field=1, type=Json, path=attributes.dimensions.width, type=I64, 400)"
Expand All @@ -386,7 +380,7 @@ mod tests {
fn test_string_term() {
let field = Field::from_field_id(1);
let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
term.append_type_and_str("red");
term.append_type_and_str_after_json_path("red");

assert_eq!(term.serialized_term(), b"\x00\x00\x00\x01jcolor\x00sred")
}
Expand All @@ -395,7 +389,7 @@ mod tests {
fn test_i64_term() {
let field = Field::from_field_id(1);
let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
term.append_type_and_fast_value(-4i64);
term.append_type_and_fast_value_after_json_path(-4i64);

assert_eq!(
term.serialized_term(),
Expand All @@ -407,7 +401,7 @@ mod tests {
fn test_u64_term() {
let field = Field::from_field_id(1);
let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
term.append_type_and_fast_value(4u64);
term.append_type_and_fast_value_after_json_path(4u64);

assert_eq!(
term.serialized_term(),
Expand All @@ -419,7 +413,7 @@ mod tests {
fn test_f64_term() {
let field = Field::from_field_id(1);
let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
term.append_type_and_fast_value(4.0f64);
term.append_type_and_fast_value_after_json_path(4.0f64);
assert_eq!(
term.serialized_term(),
b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00"
Expand All @@ -430,7 +424,7 @@ mod tests {
fn test_bool_term() {
let field = Field::from_field_id(1);
let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
term.append_type_and_fast_value(true);
term.append_type_and_fast_value_after_json_path(true);
assert_eq!(
term.serialized_term(),
b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01"
Expand Down
14 changes: 7 additions & 7 deletions src/indexer/segment_writer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -652,11 +652,11 @@ mod tests {
};

fn set_fast_val<T: FastValue>(val: T, mut term: Term) -> Term {
term.append_type_and_fast_value(val);
term.append_type_and_fast_value_after_json_path(val);
term
}
fn set_str(val: &str, mut term: Term) -> Term {
term.append_type_and_str(val);
term.append_type_and_str_after_json_path(val);
term
}

Expand Down Expand Up @@ -777,7 +777,7 @@ mod tests {
let segment_reader = searcher.segment_reader(0u32);
let inv_index = segment_reader.inverted_index(json_field).unwrap();
let mut term = term_from_json_paths(json_field, ["mykey"].into_iter(), false);
term.append_type_and_str("token");
term.append_type_and_str_after_json_path("token");
let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
assert_eq!(
term_info,
Expand Down Expand Up @@ -816,7 +816,7 @@ mod tests {
let segment_reader = searcher.segment_reader(0u32);
let inv_index = segment_reader.inverted_index(json_field).unwrap();
let mut term = term_from_json_paths(json_field, ["mykey"].into_iter(), false);
term.append_type_and_str("two tokens");
term.append_type_and_str_after_json_path("two tokens");
let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
assert_eq!(
term_info,
Expand Down Expand Up @@ -859,13 +859,13 @@ mod tests {
let term = term_from_json_paths(json_field, ["mykey", "field"].into_iter(), false);

let mut hello_term = term.clone();
hello_term.append_type_and_str("hello");
hello_term.append_type_and_str_after_json_path("hello");

let mut nothello_term = term.clone();
nothello_term.append_type_and_str("nothello");
nothello_term.append_type_and_str_after_json_path("nothello");

let mut happy_term = term.clone();
happy_term.append_type_and_str("happy");
happy_term.append_type_and_str_after_json_path("happy");

let phrase_query = PhraseQuery::new(vec![hello_term, happy_term.clone()]);
assert_eq!(searcher.search(&phrase_query, &Count).unwrap(), 1);
Expand Down
9 changes: 6 additions & 3 deletions src/query/query_parser/query_parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ use rustc_hash::FxHashMap;

use super::logical_ast::*;
use crate::core::Index;
use crate::json_utils::{convert_to_fast_value_and_append, split_json_path, term_from_json_paths};
use crate::json_utils::{
convert_to_fast_value_and_append_to_json_term, split_json_path, term_from_json_paths,
};
use crate::query::range_query::{is_type_valid_for_fastfield_range_query, RangeQuery};
use crate::query::{
AllQuery, BooleanQuery, BoostQuery, EmptyQuery, FuzzyTermQuery, Occur, PhrasePrefixQuery,
Expand Down Expand Up @@ -974,7 +976,8 @@ fn generate_literals_for_json_object(
};

// Try to convert the phrase to a fast value
if let Some(term) = convert_to_fast_value_and_append(get_term_with_path(), phrase) {
if let Some(term) = convert_to_fast_value_and_append_to_json_term(get_term_with_path(), phrase)
{
logical_literals.push(LogicalLiteral::Term(term));
}

Expand All @@ -983,7 +986,7 @@ fn generate_literals_for_json_object(
let mut token_stream = text_analyzer.token_stream(phrase);
token_stream.process(&mut |token| {
let mut term = get_term_with_path();
term.append_type_and_str(&token.text);
term.append_type_and_str_after_json_path(&token.text);
positions_and_terms.push((token.position, term.clone()));
});

Expand Down
26 changes: 22 additions & 4 deletions src/schema/term.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,20 @@ impl Term {
self.set_bytes(val.to_u64().to_be_bytes().as_ref());
}

/// Append a type marker + fast value to a term.
/// Append a string type marker + string after a JSON path.
/// This is used in JSON type to append a fast value after the path.
///
/// It will not clear existing bytes.
pub(crate) fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
pub(crate) fn append_type_and_fast_value_after_json_path<T: FastValue>(&mut self, val: T) {
debug_assert_eq!(
self.value()
.as_json_value_bytes()
.expect("expecting a json type and json path")
.as_serialized()
.len(),
0,
"JSON value bytes should be empty"
);
self.0.push(T::to_type().to_code());
let value = if T::to_type() == Type::Date {
DateTime::from_u64(val.to_u64())
Expand All @@ -178,11 +187,20 @@ impl Term {
self.0.extend(value.to_be_bytes().as_ref());
}

/// Append a string type marker + string to a term.
/// Append a string type marker + string after a JSON path.
/// This is used in JSON type to append a str after the path.
///
/// It will not clear existing bytes.
pub(crate) fn append_type_and_str(&mut self, val: &str) {
pub(crate) fn append_type_and_str_after_json_path(&mut self, val: &str) {
debug_assert_eq!(
self.value()
.as_json_value_bytes()
.expect("expecting a json type and json path")
.as_serialized()
.len(),
0,
"JSON value bytes should be empty"
);
self.0.push(Type::Str.to_code());
self.0.extend(val.as_bytes().as_ref());
}
Expand Down

0 comments on commit 44e4aeb

Please sign in to comment.