remove JsonTermWriter (quickwit-oss#2238)

* remove JsonTermWriter

remove JsonTermWriter
remove path truncation logic, add assertion

* fix json_path_writer add sep logic

common/src/json_path_writer.rs

@@ -5,6 +5,12 @@ pub const JSON_PATH_SEGMENT_SEP: u8 = 1u8;
 pub const JSON_PATH_SEGMENT_SEP_STR: &str =
     unsafe { std::str::from_utf8_unchecked(&[JSON_PATH_SEGMENT_SEP]) };
 
+/// Separates the json path and the value in
+/// a JSON term binary representation.
+pub const JSON_END_OF_PATH: u8 = 0u8;
+pub const JSON_END_OF_PATH_STR: &str =
+    unsafe { std::str::from_utf8_unchecked(&[JSON_END_OF_PATH]) };
+
 /// Create a new JsonPathWriter, that creates flattened json paths for tantivy.
 #[derive(Clone, Debug, Default)]
 pub struct JsonPathWriter {
@@ -14,6 +20,14 @@ pub struct JsonPathWriter {
 }
 
 impl JsonPathWriter {
+    pub fn with_expand_dots(expand_dots: bool) -> Self {
+        JsonPathWriter {
+            path: String::new(),
+            indices: Vec::new(),
+            expand_dots,
+        }
+    }
+
     pub fn new() -> Self {
         JsonPathWriter {
             path: String::new(),
@@ -39,8 +53,8 @@ impl JsonPathWriter {
     pub fn push(&mut self, segment: &str) {
         let len_path = self.path.len();
         self.indices.push(len_path);
-        if !self.path.is_empty() {
-            self.path.push_str(JSON_PATH_SEGMENT_SEP_STR);
+        if self.indices.len() > 1 {
+            self.path.push(JSON_PATH_SEGMENT_SEP as char);
         }
         self.path.push_str(segment);
         if self.expand_dots {
@@ -55,6 +69,12 @@ impl JsonPathWriter {
         }
     }
 
+    /// Set the end of JSON path marker.
+    #[inline]
+    pub fn set_end(&mut self) {
+        self.path.push_str(JSON_END_OF_PATH_STR);
+    }
+
     /// Remove the last segment. Does nothing if the path is empty.
     #[inline]
     pub fn pop(&mut self) {
@@ -91,6 +111,7 @@ mod tests {
     #[test]
     fn json_path_writer_test() {
         let mut writer = JsonPathWriter::new();
+        writer.set_expand_dots(false);
 
         writer.push("root");
         assert_eq!(writer.as_str(), "root");
@@ -109,4 +130,15 @@ mod tests {
         writer.push("k8s.node.id");
         assert_eq!(writer.as_str(), "root\u{1}k8s\u{1}node\u{1}id");
     }
+
+    #[test]
+    fn test_json_path_expand_dots_enabled_pop_segment() {
+        let mut json_writer = JsonPathWriter::with_expand_dots(true);
+        json_writer.push("hello");
+        assert_eq!(json_writer.as_str(), "hello");
+        json_writer.push("color.hue");
+        assert_eq!(json_writer.as_str(), "hello\x01color\x01hue");
+        json_writer.pop();
+        assert_eq!(json_writer.as_str(), "hello");
+    }
 }

common/src/lib.rs

@@ -9,7 +9,7 @@ mod byte_count;
 mod datetime;
 pub mod file_slice;
 mod group_by;
-mod json_path_writer;
+pub mod json_path_writer;
 mod serialize;
 mod vint;
 mod writer;

src/aggregation/metric/top_hits.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::net::Ipv6Addr;
 
 use columnar::{ColumnarReader, DynamicColumn};
+use common::json_path_writer::JSON_PATH_SEGMENT_SEP_STR;
 use common::DateTime;
 use regex::Regex;
 use serde::ser::SerializeMap;
@@ -15,7 +16,6 @@ use crate::aggregation::intermediate_agg_result::{
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::AggregationError;
 use crate::collector::TopNComputer;
-use crate::schema::term::JSON_PATH_SEGMENT_SEP_STR;
 use crate::schema::OwnedValue;
 use crate::{DocAddress, DocId, SegmentOrdinal};