From a55eeedaf5598c2a6e1e6943fbb8ac7e029cad30 Mon Sep 17 00:00:00 2001 From: Vladimir Kuznichenkov Date: Sat, 8 Jun 2024 18:33:46 +0300 Subject: [PATCH] Add ES Resolve index API handler This [method][1] is used by Opensearch Dashboards when user typing index name in pattern creation UI. Case that is not handled within this change is `*:*` pattern which should be converted to `*` as QuickWit do not support `:` in the index name. [1]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/indices-resolve-index-api.html --- quickwit/quickwit-lambda/src/searcher/api.rs | 1 + .../src/elasticsearch_api/filter.rs | 8 ++++ .../src/elasticsearch_api/mod.rs | 5 ++- .../elasticsearch_api/model/cat_indices.rs | 21 +++++++++ .../src/elasticsearch_api/model/mod.rs | 5 ++- .../src/elasticsearch_api/rest_handler.rs | 45 +++++++++++++++---- quickwit/quickwit-serve/src/lib.rs | 4 +- 7 files changed, 75 insertions(+), 14 deletions(-) diff --git a/quickwit/quickwit-lambda/src/searcher/api.rs b/quickwit/quickwit-lambda/src/searcher/api.rs index 14a6ee46d36..2fa60616237 100644 --- a/quickwit/quickwit-lambda/src/searcher/api.rs +++ b/quickwit/quickwit-lambda/src/searcher/api.rs @@ -86,6 +86,7 @@ fn es_compat_api( .or(es_compat_stats_handler(metastore.clone())) .or(es_compat_index_cat_indices_handler(metastore.clone())) .or(es_compat_cat_indices_handler(metastore.clone())) + .or(es_compat_resolve_index_handler(metastore.clone())) } fn index_api( diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs b/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs index b2ef41b9a41..1c6cd8df14c 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/filter.rs @@ -162,6 +162,14 @@ pub(crate) fn elastic_field_capabilities_filter() -> impl Filter< .and(json_or_empty()) } +#[utoipa::path(post, tag = "Metadata", path = "/_resolve/index/{index}")] +pub(crate) fn elastic_resolve_index_filter( +) -> impl Filter,), Error = Rejection> + Clone { + warp::path!("_elastic" / "_resolve" / "index" / String) + .and_then(extract_index_id_patterns) + .and(warp::get()) +} + #[utoipa::path(get, tag = "Count", path = "/{index}/_count")] pub(crate) fn elastic_index_count_filter( ) -> impl Filter, SearchQueryParamsCount, SearchBody), Error = Rejection> + Clone diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs b/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs index 1419d54d2be..9b21f1a2a4e 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/mod.rs @@ -38,8 +38,8 @@ pub use rest_handler::{ es_compat_cat_indices_handler, es_compat_cluster_info_handler, es_compat_delete_index_handler, es_compat_index_cat_indices_handler, es_compat_index_count_handler, es_compat_index_field_capabilities_handler, es_compat_index_multi_search_handler, - es_compat_index_search_handler, es_compat_index_stats_handler, es_compat_scroll_handler, - es_compat_search_handler, es_compat_stats_handler, + es_compat_index_search_handler, es_compat_index_stats_handler, es_compat_resolve_index_handler, + es_compat_scroll_handler, es_compat_search_handler, es_compat_stats_handler, }; use serde::{Deserialize, Serialize}; use warp::{Filter, Rejection}; @@ -79,6 +79,7 @@ pub fn elastic_api_handlers( .or(es_compat_stats_handler(metastore.clone())) .or(es_compat_index_cat_indices_handler(metastore.clone())) .or(es_compat_cat_indices_handler(metastore.clone())) + .or(es_compat_resolve_index_handler(metastore.clone())) // Register newly created handlers here. } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs index 7c73de44653..0db085b7a30 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs @@ -177,6 +177,27 @@ impl From for ElasticsearchCatIndexResponse { } } +#[derive(Debug, Clone, Default, Serialize)] +pub struct ElasticsearchResolveIndexResponse { + pub indices: Vec, +} + +#[derive(Debug, Clone, Default, Serialize)] +pub struct ElasticsearchResolveIndexEntryResponse { + pub name: String, + pub attributes: Vec, +} + +impl From for ElasticsearchResolveIndexEntryResponse { + fn from(index_metadata: IndexMetadata) -> Self { + ElasticsearchResolveIndexEntryResponse { + name: index_metadata.index_config.index_id.to_string(), + attributes: vec![Status::Open], + ..Default::default() + } + } +} + fn serialize_u64_as_string(value: &u64, serializer: S) -> Result where S: Serializer { serializer.serialize_str(&value.to_string()) diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs index 1ce635829e1..49dda496892 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs @@ -30,7 +30,10 @@ mod stats; pub use bulk_body::BulkAction; pub use bulk_query_params::ElasticBulkOptions; -pub use cat_indices::{CatIndexQueryParams, ElasticsearchCatIndexResponse}; +pub use cat_indices::{ + CatIndexQueryParams, ElasticsearchCatIndexResponse, ElasticsearchResolveIndexEntryResponse, + ElasticsearchResolveIndexResponse, +}; pub use error::{ElasticsearchError, ErrorCauseException}; pub use field_capability::{ build_list_field_request_for_es_api, convert_to_es_field_capabilities_response, diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs index ed171896728..4d8238de06c 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs @@ -50,11 +50,13 @@ use super::filter::{ elastic_field_capabilities_filter, elastic_index_cat_indices_filter, elastic_index_count_filter, elastic_index_field_capabilities_filter, elastic_index_search_filter, elastic_index_stats_filter, elastic_multi_search_filter, - elastic_scroll_filter, elastic_stats_filter, elasticsearch_filter, + elastic_resolve_index_filter, elastic_scroll_filter, elastic_stats_filter, + elasticsearch_filter, }; use super::model::{ build_list_field_request_for_es_api, convert_to_es_field_capabilities_response, CatIndexQueryParams, DeleteQueryParams, ElasticsearchCatIndexResponse, ElasticsearchError, + ElasticsearchResolveIndexEntryResponse, ElasticsearchResolveIndexResponse, ElasticsearchStatsResponse, FieldCapabilityQueryParams, FieldCapabilityRequestBody, FieldCapabilityResponse, MultiSearchHeader, MultiSearchQueryParams, MultiSearchResponse, MultiSearchSingleResponse, ScrollQueryParams, SearchBody, SearchQueryParams, @@ -133,44 +135,54 @@ pub fn es_compat_delete_index_handler( /// GET _elastic/_stats pub fn es_compat_stats_handler( - search_service: MetastoreServiceClient, + metastore_service: MetastoreServiceClient, ) -> impl Filter + Clone { elastic_stats_filter() - .and(with_arg(search_service)) + .and(with_arg(metastore_service)) .then(es_compat_stats) .map(|result| make_elastic_api_response(result, BodyFormat::default())) } /// GET _elastic/{index}/_stats pub fn es_compat_index_stats_handler( - search_service: MetastoreServiceClient, + metastore_service: MetastoreServiceClient, ) -> impl Filter + Clone { elastic_index_stats_filter() - .and(with_arg(search_service)) + .and(with_arg(metastore_service)) .then(es_compat_index_stats) .map(|result| make_elastic_api_response(result, BodyFormat::default())) } /// GET _elastic/_cat/indices pub fn es_compat_cat_indices_handler( - search_service: MetastoreServiceClient, + metastore_service: MetastoreServiceClient, ) -> impl Filter + Clone { elastic_cat_indices_filter() - .and(with_arg(search_service)) + .and(with_arg(metastore_service)) .then(es_compat_cat_indices) .map(|result| make_elastic_api_response(result, BodyFormat::default())) } /// GET _elastic/_cat/indices/{index} pub fn es_compat_index_cat_indices_handler( - search_service: MetastoreServiceClient, + metastore_service: MetastoreServiceClient, ) -> impl Filter + Clone { elastic_index_cat_indices_filter() - .and(with_arg(search_service)) + .and(with_arg(metastore_service)) .then(es_compat_index_cat_indices) .map(|result| make_elastic_api_response(result, BodyFormat::default())) } +/// GET _elastic/_resolve/index/{index} +pub fn es_compat_resolve_index_handler( + metastore_service: MetastoreServiceClient, +) -> impl Filter + Clone { + elastic_resolve_index_filter() + .and(with_arg(metastore_service)) + .then(es_compat_resolve_index) + .map(|result| make_elastic_api_response(result, BodyFormat::default())) +} + /// GET or POST _elastic/{index}/_search pub fn es_compat_index_search_handler( search_service: Arc, @@ -542,6 +554,21 @@ async fn es_compat_index_cat_indices( Ok(search_response_rest) } +async fn es_compat_resolve_index( + index_id_patterns: Vec, + mut metastore: MetastoreServiceClient, +) -> Result { + let indexes_metadata = resolve_index_patterns(&index_id_patterns, &mut metastore).await?; + let mut indices: Vec = indexes_metadata + .into_iter() + .map(|metadata| metadata.into()) + .collect(); + + indices.sort_by(|a, b| a.name.cmp(&b.name)); + + Ok(ElasticsearchResolveIndexResponse { indices }) +} + async fn es_compat_index_field_capabilities( index_id_patterns: Vec, search_params: FieldCapabilityQueryParams, diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 22c6730fc43..8ecb645d888 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -1245,8 +1245,8 @@ pub mod lambda_search_api { es_compat_cat_indices_handler, es_compat_index_cat_indices_handler, es_compat_index_count_handler, es_compat_index_field_capabilities_handler, es_compat_index_multi_search_handler, es_compat_index_search_handler, - es_compat_index_stats_handler, es_compat_scroll_handler, es_compat_search_handler, - es_compat_stats_handler, + es_compat_index_stats_handler, es_compat_resolve_index_handler, es_compat_scroll_handler, + es_compat_search_handler, es_compat_stats_handler, }; pub use crate::index_api::get_index_metadata_handler; pub use crate::rest::recover_fn;