Bound memory and disk usage of mrecordlog (#4100)

quickwit-oss · Nov 10, 2023 · 472d173 · 472d173
1 parent beabe88
commit 472d173
Show file tree

Hide file tree

Showing 14 changed files with 823 additions and 83 deletions.
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml
@@ -50,7 +50,10 @@ bytes = { version = "1", features = ["serde"] }
 bytesize = { version = "1.3.0", features = ["serde"] }
 bytestring = "1.3.0"
 chitchat = "0.7"
-chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] }
+chrono = { version = "0.4.23", default-features = false, features = [
+  "clock",
+  "std",
+] }
 clap = { version = "4.4.1", features = ["env", "string"] }
 colored = "2.0.0"
 console-subscriber = "0.1.8"
@@ -96,12 +99,19 @@ libz-sys = "1.1.8"
 lru = "0.12"
 lindera-core = "0.27.0"
 lindera-dictionary = "0.27.0"
-lindera-tokenizer = { version = "0.27.0", features = ["ipadic", "ipadic-compress", "cc-cedict", "cc-cedict-compress", "ko-dic", "ko-dic-compress"] }
+lindera-tokenizer = { version = "0.27.0", features = [
+  "cc-cedict-compress",
+  "cc-cedict",
+  "ipadic-compress",
+  "ipadic",
+  "ko-dic-compress",
+  "ko-dic",
+] }
 matches = "0.1.9"
 md5 = "0.7"
 mime_guess = "2.0.4"
 mockall = "0.11"
-mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "0d1a7aa" }
+mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "ebee0fd" }
 new_string_template = "1.4.0"
 nom = "7.1.3"
 num_cpus = "1"
@@ -113,7 +123,9 @@ opentelemetry = { version = "0.19", features = ["rt-tokio"] }
 opentelemetry-otlp = "0.12.0"
 pin-project = "1.1.0"
 pnet = { version = "0.33.0", features = ["std"] }
-postcard = { version = "1.0.4", features = ["use-std"], default-features = false}
+postcard = { version = "1.0.4", features = [
+  "use-std",
+], default-features = false }
 predicates = "3"
 prettyplease = "0.2.0"
 proc-macro2 = "1.0.50"
@@ -124,7 +136,11 @@ prost = { version = "0.11.6", default-features = false, features = [
 ] }
 prost-build = "0.11.6"
 prost-types = "0.11.6"
-pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04", default-features = false, features = ["compression", "tokio-runtime", "auth-oauth2"] }
+pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04", default-features = false, features = [
+  "auth-oauth2",
+  "compression",
+  "tokio-runtime",
+] }
 quote = "1.0.23"
 rand = "0.8"
 rand_distr = "0.4"
@@ -143,7 +159,10 @@ reqwest = { version = "0.11", default-features = false, features = [
 ] }
 rust-embed = "6.8.1"
 sea-query = { version = "0" }
-sea-query-binder = { version = "0", features = ["sqlx-postgres", "runtime-tokio-rustls",] }
+sea-query-binder = { version = "0", features = [
+  "runtime-tokio-rustls",
+  "sqlx-postgres",
+] }
 # ^1.0.184 due to serde-rs/serde#2538
 serde = { version = "1.0.184", features = ["derive", "rc"] }
 serde_json = "1.0"
@@ -152,12 +171,12 @@ serde_with = "3.4.0"
 serde_yaml = "0.9"
 siphasher = "0.3"
 sqlx = { version = "0.7", features = [
-  "runtime-tokio-rustls",
-  "postgres",
   "migrate",
+  "postgres",
+  "runtime-tokio-rustls",
   "time",
 ] }
-syn = { version = "2.0.11", features = [ "extra-traits", "full", "parsing" ]}
+syn = { version = "2.0.11", features = ["extra-traits", "full", "parsing"] }
 sync_wrapper = "0.1.2"
 tabled = { version = "0.8", features = ["color"] }
 tempfile = "3"
@@ -173,14 +192,20 @@ tokio-util = { version = "0.7", features = ["full"] }
 toml = "0.7.6"
 tonic = { version = "0.9.0", features = ["gzip"] }
 tonic-build = "0.9.0"
-tower = { version = "0.4.13", features = ["balance", "buffer", "load", "retry", "util"] }
+tower = { version = "0.4.13", features = [
+  "balance",
+  "buffer",
+  "load",
+  "retry",
+  "util",
+] }
 tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] }
 tracing = "0.1.37"
 tracing-opentelemetry = "0.19.0"
 tracing-subscriber = { version = "0.3.16", features = [
-  "time",
-  "std",
   "env-filter",
+  "std",
+  "time",
 ] }
 ttl_cache = "0.5"
 typetag = "0.2"
@@ -199,7 +224,9 @@ whichlang = { git = "https://github.com/quickwit-oss/whichlang", rev = "fe406416
 wiremock = "0.5"
 
 aws-config = "0.55.0"
-aws-credential-types = { version = "0.55.0", features = ["hardcoded-credentials"] }
+aws-credential-types = { version = "0.55.0", features = [
+  "hardcoded-credentials",
+] }
 aws-sdk-kinesis = "0.28.0"
 aws-sdk-s3 = "0.28.0"
 aws-smithy-async = "0.55.0"
@@ -209,8 +236,12 @@ aws-smithy-types = "0.55.0"
 aws-types = "0.55.0"
 
 azure_core = { version = "0.13.0", features = ["enable_reqwest_rustls"] }
-azure_storage = { version = "0.13.0", default-features = false, features = ["enable_reqwest_rustls"] }
-azure_storage_blobs = { version = "0.13.0", default-features = false, features = ["enable_reqwest_rustls"] }
+azure_storage = { version = "0.13.0", default-features = false, features = [
+  "enable_reqwest_rustls",
+] }
+azure_storage_blobs = { version = "0.13.0", default-features = false, features = [
+  "enable_reqwest_rustls",
+] }
 
 quickwit-actors = { version = "0.6.3", path = "./quickwit-actors" }
 quickwit-aws = { version = "0.6.3", path = "./quickwit-aws" }
@@ -242,10 +273,10 @@ quickwit-storage = { version = "0.6.3", path = "./quickwit-storage" }
 quickwit-telemetry = { version = "0.6.3", path = "./quickwit-telemetry" }
 
 tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "927b443", default-features = false, features = [
-  "mmap",
   "lz4-compression",
-  "zstd-compression",
+  "mmap",
   "quickwit",
+  "zstd-compression",
 ] }
 
 # This is actually not used directly the goal is to fix the version

diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs
@@ -188,7 +188,7 @@ pub struct IngestApiConfig {
     pub max_queue_memory_usage: ByteSize,
     pub max_queue_disk_usage: ByteSize,
     pub replication_factor: usize,
-    pub content_length_limit: u64,
+    pub content_length_limit: ByteSize,
 }
 
 impl Default for IngestApiConfig {
@@ -197,7 +197,7 @@ impl Default for IngestApiConfig {
             max_queue_memory_usage: ByteSize::gib(2), // TODO maybe we want more?
             max_queue_disk_usage: ByteSize::gib(4),   // TODO maybe we want more?
             replication_factor: 1,
-            content_length_limit: ByteSize::mib(10).as_u64(),
+            content_length_limit: ByteSize::mib(10),
         }
     }
 }

diff --git a/quickwit/quickwit-ingest/Cargo.toml b/quickwit/quickwit-ingest/Cargo.toml
@@ -9,6 +9,7 @@ description = "Quickwit is a cost-efficient search engine."
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 bytes = { workspace = true }
+bytesize = { workspace = true }
 dyn-clone = { workspace = true }
 flume = { workspace = true }
 futures = { workspace = true }
@@ -35,7 +36,6 @@ quickwit-config = { workspace = true }
 quickwit-proto = { workspace = true }
 
 [dev-dependencies]
-bytesize = { workspace = true }
 itertools = { workspace = true }
 mockall = { workspace = true }
 rand = { workspace = true }