Merge branch 'main' into fmassot/rest-api-ast

Dockerfile

@@ -8,7 +8,7 @@ RUN touch .gitignore_for_build_directory \
     && NODE_ENV=production make install build
 
 
-FROM rust:bullseye AS bin-builder
+FROM rust:bookworm AS bin-builder
 
 ARG CARGO_FEATURES=release-feature-set
 ARG CARGO_PROFILE=release
@@ -47,7 +47,7 @@ RUN echo "Building workspace with feature(s) '$CARGO_FEATURES' and profile '$CAR
     && find target/$CARGO_PROFILE -maxdepth 1 -perm /a+x -type f -exec mv {} /quickwit/bin \;
 
 
-FROM debian:bullseye-slim AS quickwit
+FROM debian:bookworm-slim AS quickwit
 
 LABEL org.opencontainers.image.title="Quickwit"
 LABEL maintainer="Quickwit, Inc. <[email protected]>"
@@ -56,7 +56,8 @@ LABEL org.opencontainers.image.licenses="AGPL-3.0"
 
 RUN apt-get -y update \
     && apt-get -y install ca-certificates \
-                          libssl1.1 \
+                          curl \
+                          libssl3 \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /quickwit

README.md

@@ -58,7 +58,7 @@ Cloud-native search engine for observability (logs, traces, and soon metrics!).
 - Sub-second search on cloud storage (Amazon S3, Azure Blob Storage, Google Cloud Storage, …)
 - Decoupled compute and storage, stateless indexers & searchers
 - [Grafana data source](https://github.com/quickwit-oss/quickwit-datasource)
-- Kubernetes ready - See our [helm-chart](https://quickwit.io/docs/deployment/kubernetes)
+- Kubernetes ready - See our [helm-chart](https://quickwit.io/docs/deployment/kubernetes/helm)
 - RESTful API
 
 ## Enterprise ready

config/quickwit.yaml

@@ -128,11 +128,12 @@ indexer:
 #   max_queue_disk_usage: 4GiB
 #
 # -------------------------------- Searcher settings --------------------------------
-#
+# 
 # searcher:
 #   fast_field_cache_capacity: 1G
 #   split_footer_cache_capacity: 500M
 #   max_num_concurrent_split_streams: 100
+#   partial_request_cache_capacity: 64M
 #   max_num_concurrent_split_searches: 100
 #
 # -------------------------------- Jaeger settings --------------------------------

docs/deployment/deployment-modes.md

@@ -19,7 +19,7 @@ This deployment mode is the simplest way to get started with Quickwit. Launch al
 
 ## Cluster mode (multi-node)
 
-You can deploy Quickwit on multiple nodes. We provide a [Helm chart](kubernetes.md) to help you deploy Quickwit on Kubernetes. In cluster mode, you must store your index data on a shared storage backend such as Amazon S3 or MinIO.
+You can deploy Quickwit on multiple nodes. We provide a [Helm chart](./kubernetes/helm.md) to help you deploy Quickwit on Kubernetes. In cluster mode, you must store your index data on a shared storage backend such as Amazon S3 or MinIO.
 
 ## One indexer, multiple searchers
 

docs/get-started/installation.md

@@ -17,7 +17,7 @@ Quickwit compiles to a single binary and we provide different methods to install
 
 - Linux/MacOS binaries that you can [download manually](#download) or with the [install script](#install-script)
 - [Docker image](#use-the-docker-image)
-- [Helm chart](/docs/deployment/kubernetes.md)
+- [Helm chart](../deployment/kubernetes/helm.md)
 
 ## Prerequisites
 

docs/get-started/tutorials/tutorial-aws-lambda-simple.md

@@ -138,6 +138,6 @@ Congratz! You finished this tutorial! You can level up with the following tutori
 
 ## Next steps
 
-- [Advanced Lambda tutorial](../../guides/e2e-serverless-aws-lambda) which covers an end-to-end use cases
+- [Search log service on AWS](/blog/log-search-service-for-under-7-dollars) which covers an end-to-end use case.
 - [Search REST API](../../reference/rest-api)
 - [Query language](../../reference/query-language)

quickwit/Cargo.toml

@@ -80,7 +80,7 @@ license = "AGPL-3.0-or-later"
 
 [workspace.dependencies]
 anyhow = "1"
-arc-swap = "1.6"
+arc-swap = "1.7"
 assert-json-diff = "2"
 async-compression = { version = "0.4", features = ["tokio", "gzip"] }
 async-speed-limit = "0.4"
@@ -154,7 +154,7 @@ matches = "0.1.9"
 md5 = "0.7"
 mime_guess = "2.0.4"
 mockall = "0.11"
-mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "bc6a998" }
+mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "2c593d3" }
 new_string_template = "1.4.0"
 nom = "7.1.3"
 num_cpus = "1"

quickwit/quickwit-cli/src/cli.rs

@@ -19,6 +19,7 @@
 
 use anyhow::{bail, Context};
 use clap::{arg, Arg, ArgAction, ArgMatches, Command};
+use quickwit_serve::EnvFilterReloadFn;
 use tracing::Level;
 
 use crate::index::{build_index_command, IndexCliCommand};
@@ -90,10 +91,10 @@ impl CliCommand {
         }
     }
 
-    pub async fn execute(self) -> anyhow::Result<()> {
+    pub async fn execute(self, env_filter_reload_fn: EnvFilterReloadFn) -> anyhow::Result<()> {
         match self {
             CliCommand::Index(subcommand) => subcommand.execute().await,
-            CliCommand::Run(subcommand) => subcommand.execute().await,
+            CliCommand::Run(subcommand) => subcommand.execute(env_filter_reload_fn).await,
             CliCommand::Source(subcommand) => subcommand.execute().await,
             CliCommand::Split(subcommand) => subcommand.execute().await,
             CliCommand::Tool(subcommand) => subcommand.execute().await,

quickwit/quickwit-cli/src/logger.rs

@@ -18,14 +18,15 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 use std::env;
+use std::sync::Arc;
 
 use anyhow::Context;
 use opentelemetry::sdk::propagation::TraceContextPropagator;
 use opentelemetry::sdk::trace::BatchConfig;
 use opentelemetry::sdk::{trace, Resource};
 use opentelemetry::{global, KeyValue};
 use opentelemetry_otlp::WithExportConfig;
-use quickwit_serve::BuildInfo;
+use quickwit_serve::{BuildInfo, EnvFilterReloadFn};
 use tracing::Level;
 use tracing_subscriber::fmt::time::UtcTime;
 use tracing_subscriber::prelude::*;
@@ -39,20 +40,21 @@ pub fn setup_logging_and_tracing(
     level: Level,
     ansi_colors: bool,
     build_info: &BuildInfo,
-) -> anyhow::Result<()> {
+) -> anyhow::Result<EnvFilterReloadFn> {
     #[cfg(feature = "tokio-console")]
     {
         if std::env::var_os(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY).is_some() {
             console_subscriber::init();
-            return Ok(());
+            return Ok(quickwit_serve::do_nothing_env_filter_reload_fn());
         }
     }
     let env_filter = env::var("RUST_LOG")
         .map(|_| EnvFilter::from_default_env())
         .or_else(|_| EnvFilter::try_new(format!("quickwit={level},tantivy=WARN")))
         .context("failed to set up tracing env filter")?;
     global::set_text_map_propagator(TraceContextPropagator::new());
-    let registry = tracing_subscriber::registry().with(env_filter);
+    let (reloadable_env_filter, reload_handle) = tracing_subscriber::reload::Layer::new(env_filter);
+    let registry = tracing_subscriber::registry().with(reloadable_env_filter);
     let event_format = tracing_subscriber::fmt::format()
         .with_target(true)
         .with_timer(
@@ -102,5 +104,9 @@ pub fn setup_logging_and_tracing(
             .try_init()
             .context("failed to register tracing subscriber")?;
     }
-    Ok(())
+    Ok(Arc::new(move |env_filter_def: &str| {
+        let new_env_filter = EnvFilter::try_new(env_filter_def)?;
+        reload_handle.reload(new_env_filter)?;
+        Ok(())
+    }))
 }

quickwit/quickwit-cli/src/main.rs

@@ -60,8 +60,9 @@ async fn main_impl() -> anyhow::Result<()> {
     start_jemalloc_metrics_loop();
 
     let build_info = BuildInfo::get();
-    setup_logging_and_tracing(command.default_log_level(), ansi_colors, build_info)?;
-    let return_code: i32 = if let Err(err) = command.execute().await {
+    let env_filter_reload_fn =
+        setup_logging_and_tracing(command.default_log_level(), ansi_colors, build_info)?;
+    let return_code: i32 = if let Err(err) = command.execute(env_filter_reload_fn).await {
         eprintln!("{} command failed: {:?}\n", "✘".color(RED_COLOR), err);
         1
     } else {

quickwit/quickwit-cli/src/service.rs

@@ -27,7 +27,7 @@ use quickwit_common::runtimes::RuntimesConfig;
 use quickwit_common::uri::{Protocol, Uri};
 use quickwit_config::service::QuickwitService;
 use quickwit_config::NodeConfig;
-use quickwit_serve::{serve_quickwit, BuildInfo};
+use quickwit_serve::{serve_quickwit, BuildInfo, EnvFilterReloadFn};
 use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, TelemetryEvent};
 use tokio::signal;
 use tracing::{debug, info};
@@ -74,7 +74,7 @@ impl RunCliCommand {
         })
     }
 
-    pub async fn execute(&self) -> anyhow::Result<()> {
+    pub async fn execute(&self, env_filter_reload_fn: EnvFilterReloadFn) -> anyhow::Result<()> {
         debug!(args = ?self, "run-service");
         let version_text = BuildInfo::get_version_text();
         info!("quickwit version: {version_text}");
@@ -115,6 +115,7 @@ impl RunCliCommand {
             metastore_resolver,
             storage_resolver,
             shutdown_signal,
+            env_filter_reload_fn,
         )
         .await;
         let return_code = match serve_result {

quickwit/quickwit-cli/tests/helpers.rs

@@ -158,7 +158,10 @@ impl TestEnv {
             services: Some(QuickwitService::supported_services()),
         };
         tokio::spawn(async move {
-            if let Err(error) = run_command.execute().await {
+            if let Err(error) = run_command
+                .execute(quickwit_serve::do_nothing_env_filter_reload_fn())
+                .await
+            {
                 error!(err=?error, "failed to start a quickwit server");
             }
         });

quickwit/quickwit-common/src/tower/metrics.rs

@@ -22,7 +22,7 @@ use std::task::{Context, Poll};
 use std::time::Instant;
 
 use futures::{ready, Future};
-use pin_project::pin_project;
+use pin_project::{pin_project, pinned_drop};
 use tower::{Layer, Service};
 
 use crate::metrics::{
@@ -65,6 +65,7 @@ where
             inner,
             start,
             rpc_name,
+            status: "canceled",
             requests_total: self.requests_total.clone(),
             requests_in_flight: self.requests_in_flight.clone(),
             request_duration_seconds: self.request_duration_seconds.clone(),
@@ -91,7 +92,7 @@ impl GrpcMetricsLayer {
             ),
             requests_in_flight: new_gauge_vec(
                 "grpc_requests_in_flight",
-                "Number of gRPC requests in flight.",
+                "Number of gRPC requests in-flight.",
                 subsystem,
                 &[("kind", kind)],
                 ["rpc"],
@@ -121,17 +122,34 @@ impl<S> Layer<S> for GrpcMetricsLayer {
 }
 
 /// Response future for [`PrometheusMetrics`].
-#[pin_project]
+#[pin_project(PinnedDrop)]
 pub struct ResponseFuture<F> {
     #[pin]
     inner: F,
     start: Instant,
     rpc_name: &'static str,
+    status: &'static str,
     requests_total: IntCounterVec<2>,
     requests_in_flight: IntGaugeVec<1>,
     request_duration_seconds: HistogramVec<2>,
 }
 
+#[pinned_drop]
+impl<F> PinnedDrop for ResponseFuture<F> {
+    fn drop(self: Pin<&mut Self>) {
+        let elapsed = self.start.elapsed().as_secs_f64();
+        let label_values = [self.rpc_name, self.status];
+
+        self.requests_total.with_label_values(label_values).inc();
+        self.request_duration_seconds
+            .with_label_values(label_values)
+            .observe(elapsed);
+        self.requests_in_flight
+            .with_label_values([self.rpc_name])
+            .dec();
+    }
+}
+
 impl<F, T, E> Future for ResponseFuture<F>
 where F: Future<Output = Result<T, E>>
 {
@@ -140,18 +158,7 @@ where F: Future<Output = Result<T, E>>
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
         let this = self.project();
         let response = ready!(this.inner.poll(cx));
-        let elapsed = this.start.elapsed().as_secs_f64();
-
-        let rpc_name = this.rpc_name;
-        let status = if response.is_ok() { "success" } else { "error" };
-        let label_values = [rpc_name, status];
-
-        this.requests_total.with_label_values(label_values).inc();
-        this.requests_in_flight.with_label_values([rpc_name]).dec();
-        this.request_duration_seconds
-            .with_label_values(label_values)
-            .observe(elapsed);
-
+        *this.status = if response.is_ok() { "success" } else { "error" };
         Poll::Ready(Ok(response?))
     }
 }
@@ -219,5 +226,16 @@ mod tests {
                 .get(),
             1
         );
+
+        let hello_future = hello_service.call(HelloRequest);
+        drop(hello_future);
+
+        assert_eq!(
+            layer
+                .requests_total
+                .with_label_values(["hello", "canceled"])
+                .get(),
+            1
+        );
     }
 }