diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index d1df21bb5e7..d6ffd77c6f0 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -2459,26 +2459,6 @@ dependencies = [ "encoding_rs", ] -[[package]] -name = "enum-iterator" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" -dependencies = [ - "enum-iterator-derive", -] - -[[package]] -name = "enum-iterator-derive" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.86", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -5975,6 +5955,7 @@ dependencies = [ "biscuit-auth", "futures", "http 0.2.12", + "itertools 0.13.0", "pin-project", "quickwit-common", "serde", @@ -6027,6 +6008,7 @@ dependencies = [ "opentelemetry-otlp", "predicates 3.1.2", "quickwit-actors", + "quickwit-authorize", "quickwit-cluster", "quickwit-common", "quickwit-config", @@ -6173,7 +6155,6 @@ dependencies = [ "bytesize", "chrono", "cron", - "enum-iterator", "http 0.2.12", "http-serde 1.1.3", "humantime", diff --git a/quickwit/quickwit-authorize/Cargo.toml b/quickwit/quickwit-authorize/Cargo.toml index 1b21761a408..e74b105e00d 100644 --- a/quickwit/quickwit-authorize/Cargo.toml +++ b/quickwit/quickwit-authorize/Cargo.toml @@ -14,6 +14,7 @@ tower = { workspace = true} biscuit-auth = { workspace = true, optional=true } futures = { workspace = true } http = { workspace = true } +itertools = { workspace = true } tokio-inherit-task-local = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } diff --git a/quickwit/quickwit-authorize/src/enterprise/authorization_layer.rs b/quickwit/quickwit-authorize/src/enterprise/authorization_layer.rs index ae29555ee02..891cf105203 100644 --- a/quickwit/quickwit-authorize/src/enterprise/authorization_layer.rs +++ b/quickwit/quickwit-authorize/src/enterprise/authorization_layer.rs @@ -1,21 +1,23 @@ -// Copyright (C) 2024 Quickwit, Inc. +// The Quickwit Enterprise Edition (EE) license +// Copyright (c) 2024-present Quickwit Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// With regard to the Quickwit Software: // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// This software and associated documentation files (the "Software") may only be +// used in production, if you (and any entity that you represent) hold a valid +// Quickwit Enterprise license corresponding to your usage. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// For all third party components incorporated into the Quickwit Software, those +// components are licensed under the original license provided by the owner of the +// applicable component. use std::fmt; use std::task::{Context, Poll}; diff --git a/quickwit/quickwit-authorize/src/enterprise/authorization_token_extraction_layer.rs b/quickwit/quickwit-authorize/src/enterprise/authorization_token_extraction_layer.rs index a2a9b08bfef..c32f1293200 100644 --- a/quickwit/quickwit-authorize/src/enterprise/authorization_token_extraction_layer.rs +++ b/quickwit/quickwit-authorize/src/enterprise/authorization_token_extraction_layer.rs @@ -1,21 +1,23 @@ -// Copyright (C) 2024 Quickwit, Inc. +// The Quickwit Enterprise Edition (EE) license +// Copyright (c) 2024-present Quickwit Inc. // -// Quickwit is offered under the AGPL v3.0 and as commercial software. -// For commercial licensing, contact us at hello@quickwit.io. +// With regard to the Quickwit Software: // -// AGPL: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. +// This software and associated documentation files (the "Software") may only be +// used in production, if you (and any entity that you represent) hold a valid +// Quickwit Enterprise license corresponding to your usage. // -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// For all third party components incorporated into the Quickwit Software, those +// components are licensed under the original license provided by the owner of the +// applicable component. use std::task::{Context, Poll}; diff --git a/quickwit/quickwit-authorize/src/enterprise/cli.rs b/quickwit/quickwit-authorize/src/enterprise/cli.rs new file mode 100644 index 00000000000..aa311bfebe9 --- /dev/null +++ b/quickwit/quickwit-authorize/src/enterprise/cli.rs @@ -0,0 +1,92 @@ +// Copyright (C) 2024 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::str::FromStr; + +use anyhow::Context; +use biscuit_auth::builder::{fact, string}; +use biscuit_auth::KeyPair; +use quickwit_common::QuickwitService; + +use super::AuthorizationToken; + +#[derive(Debug, Eq, PartialEq)] +pub struct GenerateAuthTokensArgs { + pub root_private_key: Option, + pub services: Vec, +} + +impl GenerateAuthTokensArgs { + fn list_services(self) -> anyhow::Result>> { + if self.services.is_empty() { + let mut default_services_set: Vec> = + vec![QuickwitService::supported_services().into_iter().collect()]; + for individual_service in QuickwitService::supported_services() { + default_services_set.push(vec![individual_service]); + } + return Ok(default_services_set); + } else { + let mut services_set: Vec> = vec![]; + for services_str in self.services { + let services = services_str + .split(",") + .map(QuickwitService::from_str) + .collect::, _>>() + .context("failed to parse quickwit service name")?; + services_set.push(services); + } + return Ok(services_set); + } + } +} + +fn generate_token_for_services( + key_pair: &KeyPair, + services: &[QuickwitService], +) -> anyhow::Result { + let mut biscuit_builder = biscuit_auth::Biscuit::builder(); + for service in services { + biscuit_builder.add_fact(fact("service", &[string(service.as_str())]))?; + } + let biscuit = biscuit_builder + .build(&key_pair) + .context("failed ot generate token")?; + Ok(AuthorizationToken::from(biscuit)) +} + +pub async fn generate_auth_tokens_cli(args: GenerateAuthTokensArgs) -> anyhow::Result<()> { + let key_pair = if let Some(private_key_hex) = &args.root_private_key { + let private_key = biscuit_auth::PrivateKey::from_bytes_hex(private_key_hex) + .context("invalid root private key")?; + biscuit_auth::KeyPair::from(&private_key) + } else { + println!("generating keys"); + biscuit_auth::KeyPair::new() + }; + println!("Private root key: {}", key_pair.private().to_bytes_hex()); + println!("Public root key: {}", key_pair.public().to_bytes_hex()); + for services in args.list_services()? { + use itertools::Itertools; + let token = generate_token_for_services(&key_pair, &services)?; + let services_str = services.iter().map(QuickwitService::as_str).join(","); + println!("--\nService token for {services_str}\n{token}"); + } + + Ok(()) +} diff --git a/quickwit/quickwit-authorize/src/enterprise/mod.rs b/quickwit/quickwit-authorize/src/enterprise/mod.rs index b8c64a5abf1..fea7b192930 100644 --- a/quickwit/quickwit-authorize/src/enterprise/mod.rs +++ b/quickwit/quickwit-authorize/src/enterprise/mod.rs @@ -27,6 +27,7 @@ use std::str::FromStr; use std::sync::{Arc, OnceLock}; use anyhow::Context; +pub mod cli; pub use authorization_layer::AuthorizationLayer; pub use authorization_token_extraction_layer::AuthorizationTokenExtractionLayer; use biscuit_auth::macros::authorizer; @@ -51,6 +52,16 @@ impl AuthorizationToken { pub fn into_biscuit(self) -> Arc { self.0.clone() } + + pub fn print(&self) -> anyhow::Result<()> { + let biscuit = &self.0; + for i in 0..biscuit.block_count() { + let block = biscuit.print_block_source(i)?; + println!("--- Block #{} ---", i + 1); + println!("{block}\n"); + } + Ok(()) + } } impl From for AuthorizationToken { @@ -61,7 +72,8 @@ impl From for AuthorizationToken { impl std::fmt::Display for AuthorizationToken { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - self.0.fmt(f) + let token_base_64 = self.0.to_base64().map_err(|_err| std::fmt::Error)?; + token_base_64.fmt(f) } } @@ -71,17 +83,19 @@ impl std::fmt::Debug for AuthorizationToken { } } -pub fn set_node_token_hex(node_token_hex: &str) -> anyhow::Result<()> { +pub fn set_node_token_base64(node_token_base64: &str) -> anyhow::Result<()> { + info!("set node token hex: {node_token_base64}"); let node_token = - AuthorizationToken::from_str(node_token_hex).context("failed to set node token")?; + AuthorizationToken::from_str(node_token_base64).context("failed to set node token")?; if NODE_TOKEN.set(node_token).is_err() { tracing::error!("node token was already initialized"); } Ok(()) } -pub fn set_root_public_key(root_key_hex: &str) -> anyhow::Result<()> { - let public_key = biscuit_auth::PublicKey::from_bytes_hex(root_key_hex) +pub fn set_root_public_key(root_key_base64: &str) -> anyhow::Result<()> { + info!(root_key = root_key_base64, "setting root public key"); + let public_key = biscuit_auth::PublicKey::from_bytes_hex(root_key_base64) .context("failed to parse root public key")?; let key_provider: Arc = Arc::new(public_key); set_root_key_provider(key_provider); @@ -106,39 +120,84 @@ impl FromStr for AuthorizationToken { fn from_str(token_base64: &str) -> Result { let root_key_provider = get_root_key_provider(); - let biscuit = Biscuit::from_base64(token_base64, root_key_provider)?; + let biscuit = Biscuit::from_base64(token_base64, root_key_provider) + .map_err(|_| AuthorizationError::InvalidToken)?; Ok(AuthorizationToken::from(biscuit)) } } const AUTHORIZATION_VALUE_PREFIX: &str = "Bearer "; -fn default_operation_authorizer( +fn default_authorizer( + request_family: RequestFamily, auth_token: &AuthorizationToken, ) -> Result { - let request_type = std::any::type_name::(); - let operation: &str = request_type.strip_suffix("Request").unwrap(); + let request_family_str = request_family.as_str(); + info!(request = request_family_str, "authorize"); let mut authorizer: Authorizer = authorizer!( r#" - operation({operation}); + request({request_family_str}); + + right($request) <- role($role), role_right($role, $request); + right($request) <- service($service), service_right($service, $request); + + service_right("control_plane", "index:read"); + service_right("control_plane", "index:write"); + service_right("control_plane", "index:admin"); + service_right("control_plane", "cluster"); + + service_right("indexer", "index:write"); + service_right("indexer", "index:read"); + service_right("indexer", "cluster"); + + service_right("searcher", "cluster"); + + service_right("janitor", "index:read"); + service_right("janitor", "cluster"); + service_right("janitor", "index:write"); // We generate the actual user role, by doing an union of the rights granted via roles. - user_right($operation) <- role($role), right($role, $operation); - user_right($operation, $resource) <- role($role), right($role, $operation, $resource); - user_right($operation) <- role("root"), operation($operation); - user_right($operation, $resource) <- role("root"), operation($operation), resource($resource); + // right($request) <- role($role), role_right($role, $request); + // right($operation, $resource) <- role($role), role_right($role, $operation, $resource); + // right($operation) <- role("root"), operation($operation); + // right($operation, $resource) <- role("root"), operation($operation), resource($resource); + // Finally we check that we have access to index1 and index2. - check all operation($operation), right($operation); + check all request($operation), right($operation); allow if true; "# ); authorizer.set_time(); - authorizer.add_token(&auth_token.0)?; + auth_token.print().unwrap(); + println!("{}", authorizer.print_world()); + authorizer + .add_token(&auth_token.0) + .map_err(|_| AuthorizationError::PermissionDenied)?; Ok(authorizer) } +#[derive(Default, Debug, Copy, Clone)] +pub enum RequestFamily { + #[default] + IndexRead, + IndexWrite, + IndexAdmin, + Cluster, +} + +impl RequestFamily { + pub fn as_str(&self) -> &'static str { + match self { + Self::IndexRead => "index:read", + Self::IndexWrite => "index:write", + Self::IndexAdmin => "index:admin", + Self::Cluster => "cluster", + } + } +} + pub trait Authorization { fn attenuate( &self, @@ -151,7 +210,11 @@ pub trait Authorization { &self, auth_token: &AuthorizationToken, ) -> Result { - default_operation_authorizer::(auth_token) + default_authorizer(Self::request_family(), auth_token) + } + + fn request_family() -> RequestFamily { + RequestFamily::default() } } @@ -164,23 +227,29 @@ pub trait StreamAuthorization { fn authorizer( auth_token: &AuthorizationToken, ) -> std::result::Result { - default_operation_authorizer::(&auth_token) + default_authorizer(Self::request_family(), &auth_token) } -} -impl From for AuthorizationError { - fn from(_token_error: biscuit_auth::error::Token) -> AuthorizationError { - AuthorizationError::InvalidToken + fn request_family() -> RequestFamily { + RequestFamily::IndexRead } } +// impl From for AuthorizationError { +// fn from(token_error: biscuit_auth::error::Token) -> AuthorizationError { +// error!(token_error=?token_error); +// AuthorizationError::InvalidToken +// } +// } + pub fn get_auth_token_from_str( authorization_header_value: &str, ) -> Result { let authorization_token_str: &str = authorization_header_value .strip_prefix(AUTHORIZATION_VALUE_PREFIX) .ok_or(AuthorizationError::InvalidToken)?; - let biscuit: Biscuit = Biscuit::from_base64(authorization_token_str, get_root_key_provider())?; + let biscuit: Biscuit = Biscuit::from_base64(authorization_token_str, get_root_key_provider()) + .map_err(|_| AuthorizationError::InvalidToken)?; Ok(AuthorizationToken::from(biscuit)) } @@ -211,8 +280,11 @@ pub fn authorize( auth_token: &AuthorizationToken, ) -> Result<(), AuthorizationError> { let mut authorizer = req.authorizer(auth_token)?; - authorizer.add_token(&auth_token.0)?; - authorizer.authorize()?; + info!("authorizer"); + authorizer + .authorize() + .map_err(|_err| AuthorizationError::PermissionDenied)?; + info!("authorize done"); Ok(()) } @@ -238,17 +310,24 @@ pub fn authorize_stream( auth_token: &AuthorizationToken, ) -> Result<(), AuthorizationError> { let mut authorizer = R::authorizer(auth_token)?; - authorizer.add_token(&auth_token.0)?; - authorizer.authorize()?; + authorizer + .add_token(&auth_token.0) + .map_err(|_| AuthorizationError::PermissionDenied)?; + authorizer + .authorize() + .map_err(|_| AuthorizationError::PermissionDenied)?; Ok(()) } pub fn authorize_request(req: &R) -> Result<(), AuthorizationError> { - let res = AUTHORIZATION_TOKEN - .try_with(|auth_token| authorize(req, auth_token)) - .unwrap_or(Err(AuthorizationError::AuthorizationTokenMissing)); info!("request authorization"); - res + let auth_token: AuthorizationToken = AUTHORIZATION_TOKEN + .try_with(|auth_token| auth_token.clone()) + .ok() + .or_else(|| NODE_TOKEN.get().cloned()) + .ok_or(AuthorizationError::AuthorizationTokenMissing)?; + info!(token=%auth_token, "auth token"); + authorize(req, &auth_token) } pub fn execute_with_authorization( diff --git a/quickwit/quickwit-cli/Cargo.toml b/quickwit/quickwit-cli/Cargo.toml index ec528e64ea9..c632f61bc58 100644 --- a/quickwit/quickwit-cli/Cargo.toml +++ b/quickwit/quickwit-cli/Cargo.toml @@ -53,6 +53,7 @@ tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } quickwit-actors = { workspace = true } +quickwit-authorize = { workspace = true, optional = true, features = ["enterprise"] } quickwit-cluster = { workspace = true } quickwit-common = { workspace = true } quickwit-config = { workspace = true } @@ -79,7 +80,7 @@ quickwit-metastore = { workspace = true, features = ["testsuite"] } quickwit-storage = { workspace = true, features = ["testsuite"] } [features] -enterprise = ["quickwit-config/enterprise", "quickwit-ingest/enterprise", "quickwit-proto/enterprise", "quickwit-serve/enterprise"] +enterprise = ["quickwit-config/enterprise", "quickwit-ingest/enterprise", "quickwit-proto/enterprise", "quickwit-serve/enterprise", "dep:quickwit-authorize"] jemalloc = ["dep:tikv-jemalloc-ctl", "dep:tikv-jemallocator"] ci-test = [] pprof = ["quickwit-serve/pprof"] diff --git a/quickwit/quickwit-cli/src/lib.rs b/quickwit/quickwit-cli/src/lib.rs index 98029541f05..93a09854e1d 100644 --- a/quickwit/quickwit-cli/src/lib.rs +++ b/quickwit/quickwit-cli/src/lib.rs @@ -28,7 +28,7 @@ use dialoguer::theme::ColorfulTheme; use dialoguer::Confirm; use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::uri::Uri; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_config::{ ConfigFormat, MetastoreConfigs, NodeConfig, SourceConfig, StorageConfigs, DEFAULT_QW_CONFIG_PATH, diff --git a/quickwit/quickwit-cli/src/service.rs b/quickwit/quickwit-cli/src/service.rs index 7c6314c0d14..476ab772366 100644 --- a/quickwit/quickwit-cli/src/service.rs +++ b/quickwit/quickwit-cli/src/service.rs @@ -27,7 +27,7 @@ use futures::future::select; use itertools::Itertools; use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::uri::{Protocol, Uri}; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_config::NodeConfig; use quickwit_serve::tcp_listener::DefaultTcpListenerResolver; use quickwit_serve::{serve_quickwit, BuildInfo, EnvFilterReloadFn}; diff --git a/quickwit/quickwit-cli/src/tool.rs b/quickwit/quickwit-cli/src/tool.rs index f5b2c512d33..41dd8167778 100644 --- a/quickwit/quickwit-cli/src/tool.rs +++ b/quickwit/quickwit-cli/src/tool.rs @@ -34,7 +34,7 @@ use quickwit_cluster::{ChannelTransport, Cluster, ClusterMember, FailureDetector use quickwit_common::pubsub::EventBroker; use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::uri::Uri; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_config::{ IndexerConfig, NodeConfig, SourceConfig, SourceInputFormat, SourceParams, TransformConfig, VecSourceParams, CLI_SOURCE_ID, @@ -66,7 +66,7 @@ use crate::{ }; pub fn build_tool_command() -> Command { - Command::new("tool") + let command = Command::new("tool") .about("Performs utility operations. Requires a node config.") .arg(config_cli_arg()) .subcommand( @@ -165,8 +165,24 @@ pub fn build_tool_command() -> Command { .display_order(2) .required(true), ]) - ) + ); + if cfg!(feature = "enterprise") { + command.subcommand( + Command::new("generate-auth-tokens") + .display_order(11) + .about("Generate authorization keys/tokens") + .args(&[ + arg!(--root "Root private key. If absent, a key pair will generated") + .display_order(1), // .required(false), + arg!(--services "Comma-separated list of services for which to generate authorization tokens") + .num_args(0..) + .display_order(2) + ]) + ) .arg_required_else_help(true) + } else { + command.arg_required_else_help(true) + } } #[derive(Debug, Eq, PartialEq)] @@ -225,6 +241,8 @@ pub enum ToolCliCommand { LocalSearch(LocalSearchArgs), Merge(MergeArgs), ExtractSplit(ExtractSplitArgs), + #[cfg(feature = "enterprise")] + GenerateAuthTokens(quickwit_authorize::cli::GenerateAuthTokensArgs), } impl ToolCliCommand { @@ -238,6 +256,8 @@ impl ToolCliCommand { "local-search" => Self::parse_local_search_args(submatches), "merge" => Self::parse_merge_args(submatches), "extract-split" => Self::parse_extract_split_args(submatches), + #[cfg(feature = "enterprise")] + "generate-auth-tokens" => Self::parse_generate_auth_tokens_args(submatches), _ => bail!("unknown tool subcommand `{subcommand}`"), } } @@ -388,6 +408,24 @@ impl ToolCliCommand { })) } + #[cfg(feature = "enterprise")] + fn parse_generate_auth_tokens_args(mut matches: ArgMatches) -> anyhow::Result { + let root_private_key = matches.remove_one::("root"); + let services_opt = matches.remove_many::("services"); + let services = if let Some(services) = services_opt { + services.collect() + } else { + Vec::new() + }; + + Ok(Self::GenerateAuthTokens( + quickwit_authorize::cli::GenerateAuthTokensArgs { + root_private_key, + services, + }, + )) + } + pub async fn execute(self) -> anyhow::Result<()> { match self { Self::GarbageCollect(args) => garbage_collect_index_cli(args).await, @@ -395,6 +433,10 @@ impl ToolCliCommand { Self::LocalSearch(args) => local_search_cli(args).await, Self::Merge(args) => merge_cli(args).await, Self::ExtractSplit(args) => extract_split_cli(args).await, + #[cfg(feature = "enterprise")] + Self::GenerateAuthTokens(args) => { + quickwit_authorize::cli::generate_auth_tokens_cli(args).await + } } } } diff --git a/quickwit/quickwit-cluster/src/lib.rs b/quickwit/quickwit-cluster/src/lib.rs index 8077d0a229a..55a44fe6b0c 100644 --- a/quickwit/quickwit-cluster/src/lib.rs +++ b/quickwit/quickwit-cluster/src/lib.rs @@ -37,7 +37,7 @@ use chitchat::{ChitchatMessage, Serializable}; pub use chitchat::{FailureDetectorConfig, KeyChangeEvent, ListenerHandle}; pub use grpc_service::cluster_grpc_server; use quickwit_common::metrics::IntCounter; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_config::NodeConfig; use quickwit_proto::indexing::CpuCapacity; use time::OffsetDateTime; diff --git a/quickwit/quickwit-cluster/src/node.rs b/quickwit/quickwit-cluster/src/node.rs index 3378e9298fd..32e960c74a8 100644 --- a/quickwit/quickwit-cluster/src/node.rs +++ b/quickwit/quickwit-cluster/src/node.rs @@ -23,7 +23,7 @@ use std::net::SocketAddr; use std::sync::Arc; use chitchat::{ChitchatId, NodeState}; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_proto::indexing::{CpuCapacity, IndexingTask}; use quickwit_proto::types::NodeIdRef; use tonic::transport::Channel; diff --git a/quickwit/quickwit-common/src/lib.rs b/quickwit/quickwit-common/src/lib.rs index a1712a4105e..443cb7ffcfc 100644 --- a/quickwit/quickwit-common/src/lib.rs +++ b/quickwit/quickwit-common/src/lib.rs @@ -21,6 +21,9 @@ mod coolid; +mod service; + +pub use service::QuickwitService; pub mod binary_heap; pub mod fs; pub mod io; diff --git a/quickwit/quickwit-config/src/service.rs b/quickwit/quickwit-common/src/service.rs similarity index 85% rename from quickwit/quickwit-config/src/service.rs rename to quickwit/quickwit-common/src/service.rs index f18bc318613..8c1a5583035 100644 --- a/quickwit/quickwit-config/src/service.rs +++ b/quickwit/quickwit-common/src/service.rs @@ -22,11 +22,11 @@ use std::fmt::Display; use std::str::FromStr; use anyhow::bail; -use enum_iterator::{all, Sequence}; +// use enum_iterator::{all, Sequence}; use itertools::Itertools; use serde::Serialize; -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Sequence)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize)] #[serde(into = "&'static str")] pub enum QuickwitService { ControlPlane, @@ -55,12 +55,20 @@ impl QuickwitService { } pub fn supported_services() -> HashSet { - all::().collect() + [ + QuickwitService::ControlPlane, + QuickwitService::Indexer, + QuickwitService::Searcher, + QuickwitService::Janitor, + QuickwitService::Metastore, + ] + .into_iter() + .collect() } } impl Display for QuickwitService { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", self.as_str()) } } diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index 0f51a309f9d..9fc913bfd2a 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -16,7 +16,6 @@ bytes = { workspace = true } bytesize = { workspace = true } chrono = { workspace = true } cron = { workspace = true } -enum-iterator = { workspace = true } http = { workspace = true } http-serde = { workspace = true } humantime = { workspace = true } diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index 2a2a6d4be60..13428b46504 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -38,7 +38,6 @@ pub mod merge_policy_config; mod metastore_config; mod node_config; mod qw_env_vars; -pub mod service; mod source_config; mod storage_config; mod templating; diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index c9a8fab1e06..53b675157b2 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -32,13 +32,13 @@ use http::HeaderMap; use quickwit_common::net::HostAddr; use quickwit_common::shared_consts::DEFAULT_SHARD_THROUGHPUT_LIMIT; use quickwit_common::uri::Uri; +use quickwit_common::QuickwitService; use quickwit_proto::indexing::CpuCapacity; use quickwit_proto::types::NodeId; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; use crate::node_config::serialize::load_node_config_with_env; -use crate::service::QuickwitService; use crate::storage_config::StorageConfigs; use crate::{ConfigFormat, MetastoreConfigs}; diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index b56026c0de2..1fda681c9fb 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -25,8 +25,8 @@ use std::time::Duration; use anyhow::{bail, Context}; use http::HeaderMap; use quickwit_common::net::{find_private_ip, get_short_hostname, Host}; -use quickwit_common::new_coolid; use quickwit_common::uri::Uri; +use quickwit_common::{new_coolid, QuickwitService}; use quickwit_proto::types::NodeId; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; @@ -34,7 +34,6 @@ use tracing::{info, warn}; use super::{GrpcConfig, RestConfig}; use crate::config_value::ConfigValue; use crate::qw_env_vars::*; -use crate::service::QuickwitService; use crate::storage_config::StorageConfigs; use crate::templating::render_config; use crate::{ @@ -372,11 +371,11 @@ impl NodeConfigBuilder { .or(self.authorization.root_public_key.as_ref()) .context("root key undefined")?; quickwit_authorize::set_root_public_key(root_public_key)?; - let node_token_hex = env_vars + let node_token_base64 = env_vars .get("QW_AUTH_NODE_TOKEN") .or(self.authorization.node_token.as_ref()) .context("root key undefined")?; - quickwit_authorize::set_node_token_hex(node_token_hex)?; + quickwit_authorize::set_node_token_base64(node_token_base64)?; Ok(()) } } @@ -435,6 +434,7 @@ impl Default for NodeConfigBuilder { ingest_api_config: IngestApiConfig::default(), jaeger_config: JaegerConfig::default(), license: license_opt, + authorization: AuthorizationConfigBuilder::default(), } } } diff --git a/quickwit/quickwit-control-plane/src/control_plane.rs b/quickwit/quickwit-control-plane/src/control_plane.rs index b24764c8ec9..f1e27c5c9b7 100644 --- a/quickwit/quickwit-control-plane/src/control_plane.rs +++ b/quickwit/quickwit-control-plane/src/control_plane.rs @@ -36,8 +36,7 @@ use quickwit_cluster::{ }; use quickwit_common::pubsub::EventSubscriber; use quickwit_common::uri::Uri; -use quickwit_common::{shared_consts, Progress}; -use quickwit_config::service::QuickwitService; +use quickwit_common::{shared_consts, Progress, QuickwitService}; use quickwit_config::{ClusterConfig, IndexConfig, IndexTemplate, SourceConfig}; use quickwit_ingest::{IngesterPool, LocalShardsUpdate}; use quickwit_metastore::{CreateIndexRequestExt, CreateIndexResponseExt, IndexMetadataResponseExt}; diff --git a/quickwit/quickwit-proto/src/authorization.rs b/quickwit/quickwit-proto/src/authorization.rs index edf0ac68b7b..df3d5f4004d 100644 --- a/quickwit/quickwit-proto/src/authorization.rs +++ b/quickwit/quickwit-proto/src/authorization.rs @@ -4,7 +4,7 @@ pub use biscuit_auth; pub use biscuit_auth::builder_ext::BuilderExt; pub use biscuit_auth::macros::*; use quickwit_authorize::{ - Authorization, AuthorizationError, AuthorizationToken, StreamAuthorization, + Authorization, AuthorizationError, AuthorizationToken, RequestFamily, StreamAuthorization, }; use crate::cluster::FetchClusterStateRequest; @@ -24,9 +24,17 @@ use crate::metastore::{ PruneShardsRequest, PublishSplitsRequest, StageSplitsRequest, UpdateSplitsDeleteOpstampRequest, }; -impl Authorization for crate::metastore::AcquireShardsRequest {} +impl Authorization for crate::metastore::AcquireShardsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexWrite + } +} -impl Authorization for crate::metastore::AddSourceRequest {} +impl Authorization for crate::metastore::AddSourceRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} impl Authorization for crate::metastore::CreateIndexRequest { fn attenuate( @@ -36,50 +44,136 @@ impl Authorization for crate::metastore::CreateIndexRequest { let mut builder = block!(r#"check if operation("create_index");"#); builder.check_expiration_date(SystemTime::now() + Duration::from_secs(60)); let biscuit = auth_token.into_biscuit(); - let new_auth_token = biscuit.append(builder)?; + let new_auth_token = biscuit + .append(builder) + .map_err(|_| AuthorizationError::PermissionDenied)?; Ok(AuthorizationToken::from(new_auth_token)) } + + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } } -impl Authorization for crate::metastore::CreateIndexTemplateRequest {} +impl Authorization for crate::metastore::CreateIndexTemplateRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::DeleteIndexRequest {} +impl Authorization for crate::metastore::DeleteIndexRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::DeleteIndexTemplatesRequest {} +impl Authorization for crate::metastore::DeleteIndexTemplatesRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::DeleteShardsRequest {} +impl Authorization for crate::metastore::DeleteShardsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::DeleteSourceRequest {} +impl Authorization for crate::metastore::DeleteSourceRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::DeleteSplitsRequest {} +impl Authorization for crate::metastore::DeleteSplitsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::FindIndexTemplateMatchesRequest {} +impl Authorization for crate::metastore::FindIndexTemplateMatchesRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexRead + } +} -impl Authorization for crate::metastore::IndexesMetadataRequest {} +impl Authorization for crate::metastore::IndexesMetadataRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexRead + } +} -impl Authorization for crate::metastore::ToggleSourceRequest {} +impl Authorization for crate::metastore::ToggleSourceRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::MarkSplitsForDeletionRequest {} +impl Authorization for crate::metastore::MarkSplitsForDeletionRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexWrite + } +} -impl Authorization for crate::metastore::ResetSourceCheckpointRequest {} +impl Authorization for crate::metastore::ResetSourceCheckpointRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for crate::metastore::UpdateIndexRequest {} +impl Authorization for crate::metastore::UpdateIndexRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexAdmin + } +} -impl Authorization for OpenObservationStreamRequest {} +impl Authorization for OpenObservationStreamRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexWrite + } +} -impl Authorization for InitShardsRequest {} +impl Authorization for InitShardsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexWrite + } +} -impl Authorization for OpenShardsRequest {} +impl Authorization for OpenShardsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexWrite + } +} -impl Authorization for FetchClusterStateRequest {} +impl Authorization for FetchClusterStateRequest { + fn request_family() -> RequestFamily { + RequestFamily::Cluster + } +} -impl Authorization for GetIndexTemplateRequest {} +impl Authorization for GetIndexTemplateRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexRead + } +} -impl Authorization for ListIndexTemplatesRequest {} +impl Authorization for ListIndexTemplatesRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexRead + } +} -impl Authorization for PruneShardsRequest {} +impl Authorization for PruneShardsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexRead + } +} -impl Authorization for ListShardsRequest {} +impl Authorization for ListShardsRequest { + fn request_family() -> RequestFamily { + RequestFamily::IndexWrite + } +} impl Authorization for ListStaleSplitsRequest {} diff --git a/quickwit/quickwit-serve/src/developer_api/debug.rs b/quickwit/quickwit-serve/src/developer_api/debug.rs index 3a1c32fbbc3..58c3ca4dd18 100644 --- a/quickwit/quickwit-serve/src/developer_api/debug.rs +++ b/quickwit/quickwit-serve/src/developer_api/debug.rs @@ -25,7 +25,7 @@ use futures::StreamExt; use glob::{MatchOptions, Pattern as GlobPattern}; use hyper::StatusCode; use quickwit_cluster::Cluster; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_proto::developer::{DeveloperService, DeveloperServiceClient, GetDebugInfoRequest}; use quickwit_proto::types::{NodeId, NodeIdRef}; use serde::Deserialize; diff --git a/quickwit/quickwit-serve/src/developer_api/server.rs b/quickwit/quickwit-serve/src/developer_api/server.rs index a06465c7efe..241b9415432 100644 --- a/quickwit/quickwit-serve/src/developer_api/server.rs +++ b/quickwit/quickwit-serve/src/developer_api/server.rs @@ -26,7 +26,7 @@ use bytes::Bytes; use bytesize::ByteSize; use quickwit_actors::Mailbox; use quickwit_cluster::Cluster; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_config::NodeConfig; use quickwit_control_plane::control_plane::{ControlPlane, GetDebugInfo}; use quickwit_ingest::{IngestRouter, Ingester}; diff --git a/quickwit/quickwit-serve/src/grpc.rs b/quickwit/quickwit-serve/src/grpc.rs index 403ae46d853..251bbb16d0d 100644 --- a/quickwit/quickwit-serve/src/grpc.rs +++ b/quickwit/quickwit-serve/src/grpc.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use bytesize::ByteSize; use quickwit_cluster::cluster_grpc_server; use quickwit_common::tower::BoxFutureInfaillible; -use quickwit_config::service::QuickwitService; +use quickwit_common::QuickwitService; use quickwit_proto::developer::DeveloperServiceClient; use quickwit_proto::indexing::IndexingServiceClient; use quickwit_proto::jaeger::storage::v1::span_reader_plugin_server::SpanReaderPluginServer; diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 62a89fee7ab..058c04128f4 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -77,8 +77,7 @@ use quickwit_common::tower::{ RateLimitLayer, RetryLayer, RetryPolicy, SmaRateEstimator, TimeoutLayer, }; use quickwit_common::uri::Uri; -use quickwit_common::{get_bool_from_env, spawn_named_task}; -use quickwit_config::service::QuickwitService; +use quickwit_common::{get_bool_from_env, spawn_named_task, QuickwitService}; use quickwit_config::{ClusterConfig, IngestApiConfig, NodeConfig}; use quickwit_control_plane::control_plane::{ControlPlane, ControlPlaneEventSubscriber}; use quickwit_control_plane::{IndexerNodeInfo, IndexerPool};