Merge branch 'main' into feat_doc_glasskube

CONTRIBUTING.md

@@ -41,6 +41,7 @@ You will be notified by email from the CI system if any issues are discovered, b
 2. Install node@20 and `npm install -g yarn`
 3. Install awslocal https://github.com/localstack/awscli-local
 4. Install protoc https://grpc.io/docs/protoc-installation/ (you may need to install the latest binaries rather than your distro's flavor)
+5. Install nextest https://nexte.st/docs/installation/pre-built-binaries/
 
 ### GitHub Codespaces
 

Dockerfile

@@ -59,7 +59,6 @@ LABEL org.opencontainers.image.licenses="AGPL-3.0"
 
 RUN apt-get -y update \
     && apt-get -y install ca-certificates \
-                          curl \
                           libssl3 \
     && rm -rf /var/lib/apt/lists/*
 

distribution/docker/ubuntu/Dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu:noble AS builder
+
+RUN apt-get update && apt-get install -y curl
+RUN curl -L https://install.quickwit.io | sh
+
+
+FROM ubuntu:noble AS quickwit
+
+LABEL org.opencontainers.image.title="Quickwit"
+LABEL maintainer="Quickwit, Inc. <[email protected]>"
+LABEL org.opencontainers.image.vendor="Quickwit, Inc."
+LABEL org.opencontainers.image.licenses="AGPL-3.0"
+
+RUN apt-get -y update \
+    && apt-get -y install ca-certificates \
+                          libssl3 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /quickwit
+RUN mkdir config qwdata
+COPY --from=builder /quickwit-v*/quickwit /usr/local/bin/quickwit
+COPY --from=builder /quickwit-v*/config/quickwit.yaml /quickwit/config/quickwit.yaml
+
+ENV QW_CONFIG=/quickwit/config/quickwit.yaml
+ENV QW_DATA_DIR=/quickwit/qwdata
+ENV QW_LISTEN_ADDRESS=0.0.0.0
+
+RUN quickwit --version
+
+ENTRYPOINT ["quickwit"]

quickwit/Cargo.toml

@@ -277,7 +277,8 @@ zstd = "0.13.0"
 aws-config = "1.2"
 aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
 aws-sdk-kinesis = "1.21"
-aws-sdk-s3 = "1.24"
+# updating aws-sdk-s3 breaks test_s3_compatible_storage_bulk_delete_multi
+aws-sdk-s3 = "=1.25"
 aws-smithy-async = "1.2"
 aws-smithy-runtime = "1.5"
 aws-smithy-types = { version = "1.2", features = ["byte-stream-poll-next"] }

quickwit/Makefile

@@ -30,11 +30,11 @@ test-all:
 	QW_S3_FORCE_PATH_STYLE_ACCESS=1 \
 	QW_TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \
 	RUST_MIN_STACK=67108864 \
-	cargo test --all-features
-	cargo test --test failpoints --features fail/failpoints
+	cargo nextest run --all-features --retries 1
+	cargo nextest run --test failpoints --features fail/failpoints
 
 test-failpoints:
-	cargo test --test failpoints --features fail/failpoints
+	cargo nextest run --test failpoints --features fail/failpoints
 
 # TODO: to be replaced by https://github.com/quickwit-oss/quickwit/issues/237
 TARGET ?= x86_64-unknown-linux-gnu

quickwit/quickwit-control-plane/src/model/shard_table.rs

@@ -32,9 +32,9 @@ use tracing::{error, info, warn};
 
 /// Limits the number of shards that can be opened for scaling up a source to 12 per minute.
 const SCALING_UP_RATE_LIMITER_SETTINGS: RateLimiterSettings = RateLimiterSettings {
-    burst_limit: 30,
-    rate_limit: ConstantRate::new(30, Duration::from_secs(60)),
-    refill_period: Duration::from_secs(2),
+    burst_limit: 12,
+    rate_limit: ConstantRate::new(5, Duration::from_secs(60)),
+    refill_period: Duration::from_secs(5),
 };
 
 /// Limits the number of shards that can be closed for scaling down a source to 2 per minute.

quickwit/quickwit-doc-mapper/src/default_doc_mapper/date_time_type.rs

@@ -71,7 +71,7 @@ impl Default for QuickwitDateTimeOptions {
 }
 
 impl QuickwitDateTimeOptions {
-    pub(crate) fn parse_json(&self, json_value: JsonValue) -> Result<TantivyValue, String> {
+    pub(crate) fn parse_json(&self, json_value: &JsonValue) -> Result<TantivyValue, String> {
         let date_time = match json_value {
             JsonValue::Number(timestamp) => {
                 // `.as_f64()` actually converts floats to integers, so we must check for integers
@@ -87,7 +87,7 @@ impl QuickwitDateTimeOptions {
                 }
             }
             JsonValue::String(date_time_str) => {
-                quickwit_datetime::parse_date_time_str(&date_time_str, &self.input_formats.0)?
+                quickwit_datetime::parse_date_time_str(date_time_str, &self.input_formats.0)?
             }
             _ => {
                 return Err(format!(
@@ -383,7 +383,7 @@ mod tests {
         let expected_timestamp = datetime!(2012-05-21 12:09:14 UTC).unix_timestamp();
         {
             let json_value = serde_json::json!("2012-05-21T12:09:14-00:00");
-            let tantivy_value = date_time_options.parse_json(json_value).unwrap();
+            let tantivy_value = date_time_options.parse_json(&json_value).unwrap();
             let date_time = match tantivy_value {
                 TantivyValue::Date(date_time) => date_time,
                 other => panic!("Expected a tantivy date time, got `{other:?}`."),
@@ -392,7 +392,7 @@ mod tests {
         }
         {
             let json_value = serde_json::json!(expected_timestamp);
-            let tantivy_value = date_time_options.parse_json(json_value).unwrap();
+            let tantivy_value = date_time_options.parse_json(&json_value).unwrap();
             let date_time = match tantivy_value {
                 TantivyValue::Date(date_time) => date_time,
                 other => panic!("Expected a tantivy date time, got `{other:?}`."),
@@ -401,7 +401,7 @@ mod tests {
         }
         {
             let json_value = serde_json::json!(expected_timestamp as f64);
-            let tantivy_value = date_time_options.parse_json(json_value).unwrap();
+            let tantivy_value = date_time_options.parse_json(&json_value).unwrap();
             let date_time = match tantivy_value {
                 TantivyValue::Date(date_time) => date_time,
                 other => panic!("Expected a tantivy date time, got `{other:?}`."),

quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs

@@ -47,9 +47,9 @@ use crate::doc_mapper::{JsonObject, Partition};
 use crate::query_builder::build_query;
 use crate::routing_expression::RoutingExpr;
 use crate::{
-    Cardinality, DocMapper, DocMapping, DocParsingError, Mode, QueryParserError, TokenizerEntry,
-    WarmupInfo, DOCUMENT_SIZE_FIELD_NAME, DYNAMIC_FIELD_NAME, FIELD_PRESENCE_FIELD_NAME,
-    SOURCE_FIELD_NAME,
+    Cardinality, DocMapper, DocMapping, DocParsingError, Mode, ModeType, QueryParserError,
+    TokenizerEntry, WarmupInfo, DOCUMENT_SIZE_FIELD_NAME, DYNAMIC_FIELD_NAME,
+    FIELD_PRESENCE_FIELD_NAME, SOURCE_FIELD_NAME,
 };
 
 const FIELD_PRESENCE_FIELD: Field = Field::from_field_id(0u32);
@@ -507,6 +507,14 @@ impl DocMapper for DefaultDocMapper {
         self.doc_mapping_uid
     }
 
+    fn validate_json_obj(&self, json_obj: &JsonObject) -> Result<(), DocParsingError> {
+        let is_strict = self.mode.mode_type() == ModeType::Strict;
+        let mut field_path = Vec::new();
+        self.field_mappings
+            .validate_from_json(json_obj, is_strict, &mut field_path)?;
+        Ok(())
+    }
+
     fn doc_from_json_obj(
         &self,
         json_obj: JsonObject,

quickwit/quickwit-doc-mapper/src/default_doc_mapper/field_mapping_entry.rs

@@ -206,7 +206,7 @@ impl BinaryFormat {
     }
 
     /// Parses the `serde_json::Value` into `tantivy::schema::Value`.
-    pub fn parse_json(&self, json_val: JsonValue) -> Result<TantivyValue, String> {
+    pub fn parse_json(&self, json_val: &JsonValue) -> Result<TantivyValue, String> {
         let byte_str = if let JsonValue::String(byte_str) = json_val {
             byte_str
         } else {
@@ -217,11 +217,11 @@ impl BinaryFormat {
         };
         let payload = match self {
             Self::Base64 => BASE64_STANDARD
-                .decode(&byte_str)
+                .decode(byte_str)
                 .map_err(|base64_decode_err| {
                     format!("expected base64 string, got `{byte_str}`: {base64_decode_err}")
                 })?,
-            Self::Hex => hex::decode(&byte_str).map_err(|hex_decode_err| {
+            Self::Hex => hex::decode(byte_str).map_err(|hex_decode_err| {
                 format!("expected hex string, got `{byte_str}`: {hex_decode_err}")
             })?,
         };