diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index c7eb4ef1f3a..1d01d98980a 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -6547,6 +6547,7 @@ dependencies = [ "hostname", "md5", "once_cell", + "quickwit-common", "reqwest", "serde", "serde_json", diff --git a/quickwit/quickwit-cli/src/logger.rs b/quickwit/quickwit-cli/src/logger.rs index ca22fc725ff..52d404b0adc 100644 --- a/quickwit/quickwit-cli/src/logger.rs +++ b/quickwit/quickwit-cli/src/logger.rs @@ -26,6 +26,7 @@ use opentelemetry::sdk::trace::BatchConfig; use opentelemetry::sdk::{trace, Resource}; use opentelemetry::{global, KeyValue}; use opentelemetry_otlp::WithExportConfig; +use quickwit_common::get_bool_from_env; use quickwit_serve::{BuildInfo, EnvFilterReloadFn}; use tracing::Level; use tracing_subscriber::fmt::time::UtcTime; @@ -43,7 +44,7 @@ pub fn setup_logging_and_tracing( ) -> anyhow::Result { #[cfg(feature = "tokio-console")] { - if std::env::var_os(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY).is_some() { + if get_bool_from_env(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY, false) { console_subscriber::init(); return Ok(quickwit_serve::do_nothing_env_filter_reload_fn()); } @@ -69,7 +70,7 @@ pub fn setup_logging_and_tracing( ); // Note on disabling ANSI characters: setting the ansi boolean on event format is insufficient. // It is thus set on layers, see https://github.com/tokio-rs/tracing/issues/1817 - if std::env::var_os(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY).is_some() { + if get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false) { let otlp_exporter = opentelemetry_otlp::new_exporter().tonic().with_env(); // In debug mode, Quickwit can generate a lot of spans, and the default queue size of 2048 // is too small. diff --git a/quickwit/quickwit-common/src/lib.rs b/quickwit/quickwit-common/src/lib.rs index 4fccfea42e0..f2a18afe0bf 100644 --- a/quickwit/quickwit-common/src/lib.rs +++ b/quickwit/quickwit-common/src/lib.rs @@ -83,26 +83,39 @@ pub fn split_file(split_id: impl Display) -> String { pub fn get_from_env(key: &str, default_value: T) -> T { if let Ok(value_str) = std::env::var(key) { if let Ok(value) = T::from_str(&value_str) { - info!(value=?value, "setting `{}` from environment", key); + info!(value=?value, "using environment variable `{key}` value"); return value; } else { - error!(value_str=%value_str, "failed to parse `{}` from environment", key); + error!(value=%value_str, "failed to parse environment variable `{key}` value"); } } - info!(value=?default_value, "setting `{}` from default", key); + info!(value=?default_value, "using environment variable `{key}` default value"); + default_value +} + +pub fn get_bool_from_env(key: &str, default_value: bool) -> bool { + if let Ok(value_str) = std::env::var(key) { + if let Some(value) = parse_bool_lenient(&value_str) { + info!(value=%value, "using environment variable `{key}` value"); + return value; + } else { + error!(value=%value_str, "failed to parse environment variable `{key}` value"); + } + } + info!(value=?default_value, "using environment variable `{key}` default value"); default_value } pub fn get_from_env_opt(key: &str) -> Option { let Some(value_str) = std::env::var(key).ok() else { - info!("{key} is not set"); + info!("environment variable `{key}` is not set"); return None; }; if let Ok(value) = T::from_str(&value_str) { - info!(value=?value, "setting `{}` from environment", key); + info!(value=?value, "using environment variable `{key}` value"); Some(value) } else { - error!(value_str=%value_str, "failed to parse `{}` from environment", key); + error!(value=%value_str, "failed to parse environment variable `{key}` value"); None } } @@ -269,6 +282,22 @@ where .unwrap() } +pub fn parse_bool_lenient(bool_str: &str) -> Option { + let trimmed_bool_str = bool_str.trim(); + + for truthy_value in ["true", "yes", "1"] { + if trimmed_bool_str.eq_ignore_ascii_case(truthy_value) { + return Some(true); + } + } + for falsy_value in ["false", "no", "0"] { + if trimmed_bool_str.eq_ignore_ascii_case(falsy_value) { + return Some(false); + } + } + None +} + #[cfg(test)] mod tests { use std::io::ErrorKind; @@ -343,4 +372,21 @@ mod tests { assert_eq!(div_ceil_u32(1, 3), 1); assert_eq!(div_ceil_u32(0, 3), 0); } + + #[test] + fn test_parse_bool_lenient() { + assert_eq!(parse_bool_lenient("true"), Some(true)); + assert_eq!(parse_bool_lenient("TRUE"), Some(true)); + assert_eq!(parse_bool_lenient("True"), Some(true)); + assert_eq!(parse_bool_lenient("yes"), Some(true)); + assert_eq!(parse_bool_lenient(" 1"), Some(true)); + + assert_eq!(parse_bool_lenient("false"), Some(false)); + assert_eq!(parse_bool_lenient("FALSE"), Some(false)); + assert_eq!(parse_bool_lenient("False"), Some(false)); + assert_eq!(parse_bool_lenient("no"), Some(false)); + assert_eq!(parse_bool_lenient("0 "), Some(false)); + + assert_eq!(parse_bool_lenient("foo"), None); + } } diff --git a/quickwit/quickwit-common/src/runtimes.rs b/quickwit/quickwit-common/src/runtimes.rs index 64884d72419..111a182b4f5 100644 --- a/quickwit/quickwit-common/src/runtimes.rs +++ b/quickwit/quickwit-common/src/runtimes.rs @@ -90,7 +90,7 @@ impl Default for RuntimesConfig { fn start_runtimes(config: RuntimesConfig) -> HashMap { let mut runtimes = HashMap::with_capacity(2); - let disable_lifo_slot: bool = crate::get_from_env("QW_DISABLE_TOKIO_LIFO_SLOT", false); + let disable_lifo_slot = crate::get_bool_from_env("QW_DISABLE_TOKIO_LIFO_SLOT", false); let mut blocking_runtime_builder = tokio::runtime::Builder::new_multi_thread(); if disable_lifo_slot { diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index 82ee8f7b0c5..76c14099d70 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -25,6 +25,7 @@ use std::str::FromStr; use anyhow::{bail, ensure, Context}; use json_comments::StripComments; use once_cell::sync::Lazy; +use quickwit_common::get_bool_from_env; use quickwit_common::net::is_valid_hostname; use quickwit_common::uri::Uri; use quickwit_proto::types::NodeIdRef; @@ -83,14 +84,16 @@ pub use crate::storage_config::{ /// Returns true if the ingest API v2 is enabled. pub fn enable_ingest_v2() -> bool { - static ENABLE_INGEST_V2: Lazy = Lazy::new(|| env::var("QW_ENABLE_INGEST_V2").is_ok()); + static ENABLE_INGEST_V2: Lazy = + Lazy::new(|| get_bool_from_env("QW_ENABLE_INGEST_V2", false)); *ENABLE_INGEST_V2 } /// Returns true if the ingest API v1 is disabled. pub fn disable_ingest_v1() -> bool { - static ENABLE_INGEST_V2: Lazy = Lazy::new(|| env::var("QW_DISABLE_INGEST_V1").is_ok()); - *ENABLE_INGEST_V2 + static DISABLE_INGEST_V1: Lazy = + Lazy::new(|| get_bool_from_env("QW_DISABLE_INGEST_V1", false)); + *DISABLE_INGEST_V1 } #[derive(utoipa::OpenApi)] diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index aaf6bfaee92..70cc2d46faf 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -29,6 +29,7 @@ use std::time::Duration; use anyhow::{bail, ensure}; use bytesize::ByteSize; use http::HeaderMap; +use quickwit_common::get_bool_from_env; use quickwit_common::net::HostAddr; use quickwit_common::uri::Uri; use quickwit_proto::indexing::CpuCapacity; @@ -122,7 +123,7 @@ impl IndexerConfig { } #[cfg(not(any(test, feature = "testsuite")))] { - true + get_bool_from_env("QW_ENABLE_OTLP_ENDPOINT", true) } } @@ -365,7 +366,7 @@ impl JaegerConfig { } #[cfg(not(any(test, feature = "testsuite")))] { - true + get_bool_from_env("QW_ENABLE_JAEGER_ENDPOINT", true) } } diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index fe06fa6117e..9bcf799a949 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -22,6 +22,7 @@ use std::{env, fmt}; use anyhow::ensure; use itertools::Itertools; +use quickwit_common::get_bool_from_env; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, EnumMap}; @@ -368,8 +369,11 @@ impl S3StorageConfig { .or_else(|| self.endpoint.clone()) } - pub fn force_path_style_access(&self) -> Option { - Some(env::var("QW_S3_FORCE_PATH_STYLE_ACCESS").is_ok() || self.force_path_style_access) + pub fn force_path_style_access(&self) -> bool { + get_bool_from_env( + "QW_S3_FORCE_PATH_STYLE_ACCESS", + self.force_path_style_access, + ) } } diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 22c6730fc43..c361e7b9153 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -70,13 +70,13 @@ use quickwit_common::pubsub::{EventBroker, EventSubscriptionHandle}; use quickwit_common::rate_limiter::RateLimiterSettings; use quickwit_common::retry::RetryParams; use quickwit_common::runtimes::RuntimesConfig; -use quickwit_common::spawn_named_task; use quickwit_common::tower::{ BalanceChannel, BoxFutureInfaillible, BufferLayer, Change, ConstantRate, EstimateRateLayer, EventListenerLayer, GrpcMetricsLayer, LoadShedLayer, RateLimitLayer, RetryLayer, RetryPolicy, SmaRateEstimator, }; use quickwit_common::uri::Uri; +use quickwit_common::{get_bool_from_env, spawn_named_task}; use quickwit_config::service::QuickwitService; use quickwit_config::{ClusterConfig, NodeConfig}; use quickwit_control_plane::control_plane::{ControlPlane, ControlPlaneEventSubscriber}; @@ -634,7 +634,7 @@ pub async fn serve_quickwit( search_job_placer, storage_resolver.clone(), event_broker.clone(), - std::env::var(DISABLE_DELETE_TASK_SERVICE_ENV_KEY).is_err(), + !get_bool_from_env(DISABLE_DELETE_TASK_SERVICE_ENV_KEY, false), ) .await .context("failed to start janitor service")?; diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index 66b17efa809..b618941fac5 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -26,6 +26,8 @@ uuid = { workspace = true } # used by reqwest. 0.8.30 has an unclear license. encoding_rs = { workspace = true } +quickwit-common = { workspace = true } + [dev-dependencies] serde_json = { workspace = true } diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index f54307be9f2..8cac61e801b 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -309,7 +309,7 @@ pub fn is_telemetry_disabled() -> bool { /// Check to see if telemetry is enabled. #[cfg(not(test))] pub fn is_telemetry_disabled() -> bool { - std::env::var_os(crate::DISABLE_TELEMETRY_ENV_KEY).is_some() + quickwit_common::get_bool_from_env(crate::DISABLE_TELEMETRY_ENV_KEY, false) } fn start_monitor_if_server_running_task(telemetry_sender: Arc) {