From 841e66c8100f31c865facecb39d4be703ce2d77b Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Wed, 28 Aug 2024 22:28:49 +0800 Subject: [PATCH] fix: config api and export metrics default database (#4633) --- config/config.md | 20 ++++++++++++-------- config/datanode.example.toml | 17 +++++++++++++++-- config/frontend.example.toml | 5 +++-- config/metasrv.example.toml | 5 +++-- config/standalone.example.toml | 5 +++-- src/cmd/src/standalone.rs | 16 +++++++++++++--- src/servers/src/export_metrics.rs | 2 +- 7 files changed, 50 insertions(+), 20 deletions(-) diff --git a/config/config.md b/config/config.md index 6d4837ae46d1..62f2e351bad8 100644 --- a/config/config.md +++ b/config/config.md @@ -160,10 +160,10 @@ | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | | `export_metrics.enable` | Bool | `false` | whether enable export metrics. | | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. | -| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself | +| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
You must create the database before enabling it. | | `export_metrics.self_import.db` | String | `None` | -- | | `export_metrics.remote_write` | -- | -- | -- | -| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. | +| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. | | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. | | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. | | `tracing.tokio_console_addr` | String | `None` | The tokio console address. | @@ -245,10 +245,10 @@ | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | | `export_metrics.enable` | Bool | `false` | whether enable export metrics. | | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. | -| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself | +| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
You must create the database before enabling it. | | `export_metrics.self_import.db` | String | `None` | -- | | `export_metrics.remote_write` | -- | -- | -- | -| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. | +| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. | | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. | | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. | | `tracing.tokio_console_addr` | String | `None` | The tokio console address. | @@ -309,10 +309,10 @@ | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | | `export_metrics.enable` | Bool | `false` | whether enable export metrics. | | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. | -| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself | +| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
You must create the database before enabling it. | | `export_metrics.self_import.db` | String | `None` | -- | | `export_metrics.remote_write` | -- | -- | -- | -| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. | +| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. | | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. | | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. | | `tracing.tokio_console_addr` | String | `None` | The tokio console address. | @@ -333,6 +333,10 @@ | `rpc_runtime_size` | Integer | `None` | Deprecated, use `grpc.runtime_size` instead. | | `rpc_max_recv_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_recv_message_size` instead. | | `rpc_max_send_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_send_message_size` instead. | +| `http` | -- | -- | The HTTP server options. | +| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. | +| `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. | +| `http.body_limit` | String | `64MB` | HTTP request body limit.
The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
Set to 0 to disable limit. | | `grpc` | -- | -- | The gRPC server options. | | `grpc.addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. | | `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,
and used for connections from outside the host | @@ -453,10 +457,10 @@ | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. | | `export_metrics.enable` | Bool | `false` | whether enable export metrics. | | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. | -| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself | +| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
You must create the database before enabling it. | | `export_metrics.self_import.db` | String | `None` | -- | | `export_metrics.remote_write` | -- | -- | -- | -| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. | +| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. | | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. | | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. | | `tracing.tokio_console_addr` | String | `None` | The tokio console address. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index a7357f934566..d1c370819ec7 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -39,6 +39,18 @@ rpc_max_recv_message_size = "512MB" ## +toml2docs:none-default rpc_max_send_message_size = "512MB" + +## The HTTP server options. +[http] +## The address to bind the HTTP server. +addr = "127.0.0.1:4000" +## HTTP request timeout. Set to 0 to disable timeout. +timeout = "30s" +## HTTP request body limit. +## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`. +## Set to 0 to disable limit. +body_limit = "64MB" + ## The gRPC server options. [grpc] ## The address to bind the gRPC server. @@ -552,12 +564,13 @@ enable = false write_interval = "30s" ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself +## You must create the database before enabling it. [export_metrics.self_import] ## +toml2docs:none-default -db = "information_schema" +db = "greptime_metrics" [export_metrics.remote_write] -## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. +## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. url = "" ## HTTP headers of Prometheus remote-write carry. diff --git a/config/frontend.example.toml b/config/frontend.example.toml index d59719c8c1d2..a3b8ef366f8d 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -199,12 +199,13 @@ enable = false write_interval = "30s" ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself +## You must create the database before enabling it. [export_metrics.self_import] ## +toml2docs:none-default -db = "information_schema" +db = "greptime_metrics" [export_metrics.remote_write] -## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. +## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. url = "" ## HTTP headers of Prometheus remote-write carry. diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index 57be17b52021..03be77c07097 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -186,12 +186,13 @@ enable = false write_interval = "30s" ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself +## You must create the database before enabling it. [export_metrics.self_import] ## +toml2docs:none-default -db = "information_schema" +db = "greptime_metrics" [export_metrics.remote_write] -## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. +## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. url = "" ## HTTP headers of Prometheus remote-write carry. diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 7a9a09dc5ab4..85ef2e3d00b3 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -601,12 +601,13 @@ enable = false write_interval = "30s" ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself +## You must create the database before enabling it. [export_metrics.self_import] ## +toml2docs:none-default -db = "information_schema" +db = "greptime_metrics" [export_metrics.remote_write] -## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. +## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. url = "" ## HTTP headers of Prometheus remote-write carry. diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 97899120f9b4..83875dff06d4 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -178,6 +178,16 @@ impl Configurable for StandaloneOptions { } } +/// The [`StandaloneOptions`] is only defined in cmd crate, +/// we don't want to make `frontend` depends on it, so impl [`Into`] +/// rather than [`From`]. +#[allow(clippy::from_over_into)] +impl Into for StandaloneOptions { + fn into(self) -> FrontendOptions { + self.frontend_options() + } +} + impl StandaloneOptions { pub fn frontend_options(&self) -> FrontendOptions { let cloned_opts = self.clone(); @@ -510,7 +520,7 @@ impl StartCommand { .build(), ); let wal_options_allocator = Arc::new(WalOptionsAllocator::new( - opts.wal.into(), + opts.wal.clone().into(), kv_backend.clone(), )); let table_meta_allocator = Arc::new(TableMetadataAllocator::new( @@ -533,7 +543,7 @@ impl StartCommand { .await?; let mut frontend = FrontendBuilder::new( - fe_opts.clone(), + fe_opts, kv_backend.clone(), layered_cache_registry.clone(), catalog_manager.clone(), @@ -561,7 +571,7 @@ impl StartCommand { let (tx, _rx) = broadcast::channel(1); - let servers = Services::new(fe_opts, Arc::new(frontend.clone()), plugins) + let servers = Services::new(opts, Arc::new(frontend.clone()), plugins) .build() .await .context(StartFrontendSnafu)?; diff --git a/src/servers/src/export_metrics.rs b/src/servers/src/export_metrics.rs index 36577d99a643..4834940c0e1f 100644 --- a/src/servers/src/export_metrics.rs +++ b/src/servers/src/export_metrics.rs @@ -59,7 +59,7 @@ pub struct SelfImportOption { impl Default for SelfImportOption { fn default() -> Self { Self { - db: "information_schema".to_string(), + db: "greptime_metrics".to_string(), } } }