From a86914a7433be87995f3e1c7f382b24eeffe1b35 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Wed, 4 Sep 2024 11:04:51 +0800 Subject: [PATCH] refactor: make `init_global_logging()` clean and add `log_format` (#4657) refactor: refine the code logic of init_global_logging and add json output format --- Cargo.lock | 13 ++ config/config.md | 15 +- config/datanode.example.toml | 7 +- config/flownode.example.toml | 5 +- config/frontend.example.toml | 5 +- config/metasrv.example.toml | 5 +- config/standalone.example.toml | 5 +- src/common/telemetry/Cargo.toml | 2 +- src/common/telemetry/src/logging.rs | 223 ++++++++++++++++++---------- tests-integration/tests/http.rs | 3 +- 10 files changed, 190 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18d16b268239..47b22a44d7d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12441,6 +12441,16 @@ dependencies = [ "web-time 0.2.4", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -12451,12 +12461,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log 0.2.0", + "tracing-serde", ] [[package]] diff --git a/config/config.md b/config/config.md index 5e5d8e7f5dea..83bc85612cd0 100644 --- a/config/config.md +++ b/config/config.md @@ -152,11 +152,12 @@ | `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.
Only available for `partition_tree` memtable. | | `region_engine.file` | -- | -- | Enable the file engine. | | `logging` | -- | -- | The logging options. | -| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. | +| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. | | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. | | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. | | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. | +| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- | | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | @@ -237,11 +238,12 @@ | `datanode.client.connect_timeout` | String | `10s` | -- | | `datanode.client.tcp_nodelay` | Bool | `true` | -- | | `logging` | -- | -- | The logging options. | -| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. | +| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. | | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. | | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. | | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. | +| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- | | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | @@ -301,11 +303,12 @@ | `wal.backoff_base` | Integer | `2` | Exponential backoff rate, i.e. next backoff = base * current backoff. | | `wal.backoff_deadline` | String | `5mins` | Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate. | | `logging` | -- | -- | The logging options. | -| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. | +| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. | | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. | | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. | | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. | +| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- | | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | @@ -449,11 +452,12 @@ | `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.
Only available for `partition_tree` memtable. | | `region_engine.file` | -- | -- | Enable the file engine. | | `logging` | -- | -- | The logging options. | -| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. | +| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. | | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. | | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. | | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. | +| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- | | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | @@ -494,11 +498,12 @@ | `heartbeat.interval` | String | `3s` | Interval for sending heartbeat messages to the metasrv. | | `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. | | `logging` | -- | -- | The logging options. | -| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. | +| `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. | | `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. | | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. | | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. | | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. | +| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- | | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 48f37b62ea3c..8b55d6e5331a 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -336,7 +336,7 @@ credential_path = "test" ## The credential of the google cloud storage. ## **It's only used when the storage type is `Gcs`**. ## +toml2docs:none-default -credential= "base64-credential" +credential = "base64-credential" ## The container of the azure account. ## **It's only used when the storage type is `Azblob`**. @@ -545,7 +545,7 @@ fork_dictionary_bytes = "1GiB" ## The logging options. [logging] -## The directory to store the log files. +## The directory to store the log files. If set to empty, logs will not be written to files. dir = "/tmp/greptimedb/logs" ## The log level. Can be `info`/`debug`/`warn`/`error`. @@ -561,6 +561,9 @@ otlp_endpoint = "http://localhost:4317" ## Whether to append logs to stdout. append_stdout = true +## The log format. Can be `text`/`json`. +log_format = "text" + ## The percentage of tracing will be sampled and exported. ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0 diff --git a/config/flownode.example.toml b/config/flownode.example.toml index 443edb0303f5..d5640062f701 100644 --- a/config/flownode.example.toml +++ b/config/flownode.example.toml @@ -59,7 +59,7 @@ retry_interval = "3s" ## The logging options. [logging] -## The directory to store the log files. +## The directory to store the log files. If set to empty, logs will not be written to files. dir = "/tmp/greptimedb/logs" ## The log level. Can be `info`/`debug`/`warn`/`error`. @@ -75,6 +75,9 @@ otlp_endpoint = "http://localhost:4317" ## Whether to append logs to stdout. append_stdout = true +## The log format. Can be `text`/`json`. +log_format = "text" + ## The percentage of tracing will be sampled and exported. ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0 diff --git a/config/frontend.example.toml b/config/frontend.example.toml index a3b8ef366f8d..e5a7f5af898d 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -166,7 +166,7 @@ tcp_nodelay = true ## The logging options. [logging] -## The directory to store the log files. +## The directory to store the log files. If set to empty, logs will not be written to files. dir = "/tmp/greptimedb/logs" ## The log level. Can be `info`/`debug`/`warn`/`error`. @@ -182,6 +182,9 @@ otlp_endpoint = "http://localhost:4317" ## Whether to append logs to stdout. append_stdout = true +## The log format. Can be `text`/`json`. +log_format = "text" + ## The percentage of tracing will be sampled and exported. ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0 diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index 03be77c07097..dc5f09116646 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -153,7 +153,7 @@ backoff_deadline = "5mins" ## The logging options. [logging] -## The directory to store the log files. +## The directory to store the log files. If set to empty, logs will not be written to files. dir = "/tmp/greptimedb/logs" ## The log level. Can be `info`/`debug`/`warn`/`error`. @@ -169,6 +169,9 @@ otlp_endpoint = "http://localhost:4317" ## Whether to append logs to stdout. append_stdout = true +## The log format. Can be `text`/`json`. +log_format = "text" + ## The percentage of tracing will be sampled and exported. ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0 diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 6ca7d917d39a..b065e1a09d57 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -589,7 +589,7 @@ fork_dictionary_bytes = "1GiB" ## The logging options. [logging] -## The directory to store the log files. +## The directory to store the log files. If set to empty, logs will not be written to files. dir = "/tmp/greptimedb/logs" ## The log level. Can be `info`/`debug`/`warn`/`error`. @@ -605,6 +605,9 @@ otlp_endpoint = "http://localhost:4317" ## Whether to append logs to stdout. append_stdout = true +## The log format. Can be `text`/`json`. +log_format = "text" + ## The percentage of tracing will be sampled and exported. ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0 diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml index 3cf2a36a4020..20fc52a763f7 100644 --- a/src/common/telemetry/Cargo.toml +++ b/src/common/telemetry/Cargo.toml @@ -34,4 +34,4 @@ tracing = "0.1" tracing-appender = "0.2" tracing-log = "0.1" tracing-opentelemetry = "0.22.0" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] } diff --git a/src/common/telemetry/src/logging.rs b/src/common/telemetry/src/logging.rs index 04766c20dbce..7047a8993c61 100644 --- a/src/common/telemetry/src/logging.rs +++ b/src/common/telemetry/src/logging.rs @@ -35,15 +35,37 @@ use crate::tracing_sampler::{create_sampler, TracingSampleOptions}; pub const DEFAULT_OTLP_ENDPOINT: &str = "http://localhost:4317"; +/// The logging options that used to initialize the logger. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default)] pub struct LoggingOptions { + /// The directory to store log files. If not set, logs will be written to stdout. pub dir: String, + + /// The log level that can be one of "trace", "debug", "info", "warn", "error". Default is "info". pub level: Option, + + /// The log format that can be one of "json" or "text". Default is "text". + pub log_format: LogFormat, + + /// Whether to append logs to stdout. Default is true. + pub append_stdout: bool, + + /// Whether to enable tracing with OTLP. Default is false. pub enable_otlp_tracing: bool, + + /// The endpoint of OTLP. Default is "http://localhost:4317". pub otlp_endpoint: Option, + + /// The tracing sample ratio. pub tracing_sample_ratio: Option, - pub append_stdout: bool, +} + +#[derive(Clone, Debug, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogFormat { + Json, + Text, } impl PartialEq for LoggingOptions { @@ -64,6 +86,7 @@ impl Default for LoggingOptions { Self { dir: "/tmp/greptimedb/logs".to_string(), level: None, + log_format: LogFormat::Text, enable_otlp_tracing: false, otlp_endpoint: None, tracing_sample_ratio: None, @@ -128,62 +151,97 @@ pub fn init_global_logging( let mut guards = vec![]; START.call_once(|| { - let dir = &opts.dir; - let level = &opts.level; - let enable_otlp_tracing = opts.enable_otlp_tracing; - // Enable log compatible layer to convert log record to tracing span. LogTracer::init().expect("log tracer must be valid"); - // stdout log layer. + // Configure the stdout logging layer. let stdout_logging_layer = if opts.append_stdout { - let (stdout_writer, stdout_guard) = tracing_appender::non_blocking(std::io::stdout()); - guards.push(stdout_guard); + let (writer, guard) = tracing_appender::non_blocking(std::io::stdout()); + guards.push(guard); + + if opts.log_format == LogFormat::Json { + Some( + Layer::new() + .json() + .with_writer(writer) + .with_ansi(atty::is(atty::Stream::Stdout)) + .boxed(), + ) + } else { + Some( + Layer::new() + .with_writer(writer) + .with_ansi(atty::is(atty::Stream::Stdout)) + .boxed(), + ) + } + } else { + None + }; - Some( - Layer::new() - .with_writer(stdout_writer) - .with_ansi(atty::is(atty::Stream::Stdout)), - ) + // Configure the file logging layer with rolling policy. + let file_logging_layer = if !opts.dir.is_empty() { + let rolling_appender = + RollingFileAppender::new(Rotation::HOURLY, &opts.dir, "greptimedb"); + let (writer, guard) = tracing_appender::non_blocking(rolling_appender); + guards.push(guard); + + if opts.log_format == LogFormat::Json { + Some( + Layer::new() + .json() + .with_writer(writer) + .with_ansi(false) + .boxed(), + ) + } else { + Some(Layer::new().with_writer(writer).with_ansi(false).boxed()) + } } else { None }; - // file log layer. - let rolling_appender = RollingFileAppender::new(Rotation::HOURLY, dir, app_name); - let (rolling_writer, rolling_writer_guard) = - tracing_appender::non_blocking(rolling_appender); - let file_logging_layer = Layer::new().with_writer(rolling_writer).with_ansi(false); - guards.push(rolling_writer_guard); - - // error file log layer. - let err_rolling_appender = - RollingFileAppender::new(Rotation::HOURLY, dir, format!("{}-{}", app_name, "err")); - let (err_rolling_writer, err_rolling_writer_guard) = - tracing_appender::non_blocking(err_rolling_appender); - let err_file_logging_layer = Layer::new() - .with_writer(err_rolling_writer) - .with_ansi(false); - guards.push(err_rolling_writer_guard); + // Configure the error file logging layer with rolling policy. + let err_file_logging_layer = if !opts.dir.is_empty() { + let rolling_appender = + RollingFileAppender::new(Rotation::HOURLY, &opts.dir, "greptimedb-err"); + let (writer, guard) = tracing_appender::non_blocking(rolling_appender); + guards.push(guard); + + if opts.log_format == LogFormat::Json { + Some( + Layer::new() + .json() + .with_writer(writer) + .with_ansi(false) + .with_filter(filter::LevelFilter::ERROR) + .boxed(), + ) + } else { + Some( + Layer::new() + .with_writer(writer) + .with_ansi(false) + .with_filter(filter::LevelFilter::ERROR) + .boxed(), + ) + } + } else { + None + }; // resolve log level settings from: // - options from command line or config files // - environment variable: RUST_LOG // - default settings - let rust_log_env = std::env::var(EnvFilter::DEFAULT_ENV).ok(); - let targets_string = level + let filter = opts + .level .as_deref() - .or(rust_log_env.as_deref()) - .unwrap_or(DEFAULT_LOG_TARGETS); - let filter = targets_string + .or(env::var(EnvFilter::DEFAULT_ENV).ok().as_deref()) + .unwrap_or(DEFAULT_LOG_TARGETS) .parse::() .expect("error parsing log level string"); - let sampler = opts - .tracing_sample_ratio - .as_ref() - .map(create_sampler) - .map(Sampler::ParentBased) - .unwrap_or(Sampler::ParentBased(Box::new(Sampler::AlwaysOn))); + // Must enable 'tokio_unstable' cfg to use this feature. // For example: `RUSTFLAGS="--cfg tokio_unstable" cargo run -F common-telemetry/console -- standalone start` #[cfg(feature = "tokio-console")] @@ -204,18 +262,15 @@ pub fn init_global_logging( None }; - let stdout_logging_layer = stdout_logging_layer.map(|x| x.with_filter(filter.clone())); - - let file_logging_layer = file_logging_layer.with_filter(filter); - Registry::default() + .with(filter) .with(tokio_console_layer) .with(stdout_logging_layer) .with(file_logging_layer) - .with(err_file_logging_layer.with_filter(filter::LevelFilter::ERROR)) + .with(err_file_logging_layer) }; - // consume the `tracing_opts`, to avoid "unused" warnings + // consume the `tracing_opts` to avoid "unused" warnings. let _ = tracing_opts; #[cfg(not(feature = "tokio-console"))] @@ -223,46 +278,54 @@ pub fn init_global_logging( .with(filter) .with(stdout_logging_layer) .with(file_logging_layer) - .with(err_file_logging_layer.with_filter(filter::LevelFilter::ERROR)); + .with(err_file_logging_layer); - if enable_otlp_tracing { + if opts.enable_otlp_tracing { global::set_text_map_propagator(TraceContextPropagator::new()); - // otlp exporter + + let sampler = opts + .tracing_sample_ratio + .as_ref() + .map(create_sampler) + .map(Sampler::ParentBased) + .unwrap_or(Sampler::ParentBased(Box::new(Sampler::AlwaysOn))); + + let trace_config = opentelemetry_sdk::trace::config() + .with_sampler(sampler) + .with_resource(opentelemetry_sdk::Resource::new(vec![ + KeyValue::new(resource::SERVICE_NAME, app_name.to_string()), + KeyValue::new( + resource::SERVICE_INSTANCE_ID, + node_id.unwrap_or("none".to_string()), + ), + KeyValue::new(resource::SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + KeyValue::new(resource::PROCESS_PID, std::process::id().to_string()), + ])); + + let exporter = opentelemetry_otlp::new_exporter().tonic().with_endpoint( + opts.otlp_endpoint + .as_ref() + .map(|e| { + if e.starts_with("http") { + e.to_string() + } else { + format!("http://{}", e) + } + }) + .unwrap_or(DEFAULT_OTLP_ENDPOINT.to_string()), + ); + let tracer = opentelemetry_otlp::new_pipeline() .tracing() - .with_exporter( - opentelemetry_otlp::new_exporter().tonic().with_endpoint( - opts.otlp_endpoint - .as_ref() - .map(|e| { - if e.starts_with("http") { - e.to_string() - } else { - format!("http://{}", e) - } - }) - .unwrap_or(DEFAULT_OTLP_ENDPOINT.to_string()), - ), - ) - .with_trace_config( - opentelemetry_sdk::trace::config() - .with_sampler(sampler) - .with_resource(opentelemetry_sdk::Resource::new(vec![ - KeyValue::new(resource::SERVICE_NAME, app_name.to_string()), - KeyValue::new( - resource::SERVICE_INSTANCE_ID, - node_id.unwrap_or("none".to_string()), - ), - KeyValue::new(resource::SERVICE_VERSION, env!("CARGO_PKG_VERSION")), - KeyValue::new(resource::PROCESS_PID, std::process::id().to_string()), - ])), - ) + .with_exporter(exporter) + .with_trace_config(trace_config) .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("otlp tracer install failed"); - let tracing_layer = Some(tracing_opentelemetry::layer().with_tracer(tracer)); - let subscriber = subscriber.with(tracing_layer); - tracing::subscriber::set_global_default(subscriber) - .expect("error setting global tracing subscriber"); + + tracing::subscriber::set_global_default( + subscriber.with(tracing_opentelemetry::layer().with_tracer(tracer)), + ) + .expect("error setting global tracing subscriber"); } else { tracing::subscriber::set_global_default(subscriber) .expect("error setting global tracing subscriber"); diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 4d9c6ceb871c..7b4164023eb6 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -823,8 +823,8 @@ max_retry_times = 3 retry_delay = "500ms" [logging] -enable_otlp_tracing = false append_stdout = true +enable_otlp_tracing = false [[region_engine]] @@ -883,6 +883,7 @@ write_interval = "30s" fn drop_lines_with_inconsistent_results(input: String) -> String { let inconsistent_results = [ "dir =", + "log_format =", "data_home =", "bucket =", "root =",