feat(bloom-filter): add basic bloom filter creator (Part 1) (#5177)

* feat(bloom-filter): add a simple bloom filter creator (Part 1)

Signed-off-by: Zhenchi <[email protected]>

* fix: clippy

Signed-off-by: Zhenchi <[email protected]>

* fix: header

Signed-off-by: Zhenchi <[email protected]>

* docs: add format comment

Signed-off-by: Zhenchi <[email protected]>

---------

Signed-off-by: Zhenchi <[email protected]>

src/index/Cargo.toml

@@ -17,6 +17,7 @@ common-error.workspace = true
 common-macro.workspace = true
 common-runtime.workspace = true
 common-telemetry.workspace = true
+fastbloom = "0.8"
 fst.workspace = true
 futures.workspace = true
 greptime-proto.workspace = true
@@ -26,6 +27,7 @@ prost.workspace = true
 regex.workspace = true
 regex-automata.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 snafu.workspace = true
 tantivy = { version = "0.22", features = ["zstd-compression"] }
 tantivy-jieba = "0.11.0"

src/index/src/bloom_filter.rs

@@ -0,0 +1,53 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+
+pub mod creator;
+mod error;
+
+pub type Bytes = Vec<u8>;
+pub type BytesRef<'a> = &'a [u8];
+
+/// The Meta information of the bloom filter stored in the file.
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BloomFilterMeta {
+    /// The number of rows per segment.
+    pub rows_per_segment: usize,
+
+    /// The number of segments.
+    pub seg_count: usize,
+
+    /// The number of total rows.
+    pub row_count: usize,
+
+    /// The size of the bloom filter excluding the meta information.
+    pub bloom_filter_segments_size: usize,
+
+    /// Offset and size of bloom filters in the file.
+    pub bloom_filter_segments: Vec<BloomFilterSegmentLocation>,
+}
+
+/// The location of the bloom filter segment in the file.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BloomFilterSegmentLocation {
+    /// The offset of the bloom filter segment in the file.
+    pub offset: u64,
+
+    /// The size of the bloom filter segment in the file.
+    pub size: u64,
+
+    /// The number of elements in the bloom filter segment.
+    pub elem_count: usize,
+}