From 31490d08bac4a976cc3eaea0a5d903baf2ed134f Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Wed, 31 Jan 2024 17:44:09 +0800 Subject: [PATCH] handle empty indices in cat indices --- .../elasticsearch_api/model/cat_indices.rs | 21 ++++++---- .../src/elasticsearch_api/rest_handler.rs | 41 +++++++++---------- .../es_compatibility/0021-cat-indices.yaml | 6 +++ .../_setup.elasticsearch.yaml | 14 +++++++ .../es_compatibility/_setup.quickwit.yaml | 14 +++++++ .../_teardown.elasticsearch.yaml | 3 ++ .../es_compatibility/_teardown.quickwit.yaml | 5 +++ 7 files changed, 75 insertions(+), 29 deletions(-) diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs index e7b98166d97..9faf21921a7 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs @@ -21,7 +21,7 @@ use std::collections::HashSet; use std::ops::AddAssign; use hyper::StatusCode; -use quickwit_metastore::SplitMetadata; +use quickwit_metastore::{IndexMetadata, SplitMetadata}; use serde::{Deserialize, Serialize, Serializer}; use super::ElasticsearchError; @@ -135,13 +135,6 @@ impl ElasticsearchCatIndexResponse { } impl AddAssign for ElasticsearchCatIndexResponse { fn add_assign(&mut self, rhs: Self) { - // pri and rep are always 1, so we can just overwrite them - self.pri = rhs.pri; - self.rep = rhs.rep; - // Set index, since this may be a default entry - self.index = rhs.index; - self.uuid = rhs.uuid; - self.health += rhs.health; self.status += rhs.status; self.docs_count += rhs.docs_count; @@ -152,6 +145,18 @@ impl AddAssign for ElasticsearchCatIndexResponse { } } +impl From for ElasticsearchCatIndexResponse { + fn from(index_metadata: IndexMetadata) -> Self { + ElasticsearchCatIndexResponse { + uuid: index_metadata.index_uid.to_string(), + index: index_metadata.index_config.index_id.to_string(), + pri: "1".to_string(), + rep: "1".to_string(), + ..Default::default() + } + } +} + impl From for ElasticsearchCatIndexResponse { fn from(split_metadata: SplitMetadata) -> Self { ElasticsearchCatIndexResponse { diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs index 4877c335dd2..529254126aa 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs @@ -426,21 +426,23 @@ async fn es_compat_index_cat_indices( ) -> Result, ElasticsearchError> { query_params.validate()?; let indexes_metadata = resolve_index_patterns(&index_id_patterns, &mut metastore).await?; - // Index id to index uid mapping - let index_uid_to_index_id: HashMap = indexes_metadata + let mut index_id_to_resp: HashMap = indexes_metadata .iter() - .map(|metadata| (metadata.index_uid.clone(), metadata.index_id().to_owned())) + .map(|metadata| (metadata.index_uid.to_owned(), metadata.clone().into())) .collect(); - let index_uids = indexes_metadata - .into_iter() - .map(|index_metadata| index_metadata.index_uid) - .collect_vec(); - // calling into the search module is not necessary, but reuses established patterns - let splits_metadata = list_all_splits(index_uids, &mut metastore).await?; + let splits_metadata = { + let index_uids = indexes_metadata + .into_iter() + .map(|index_metadata| index_metadata.index_uid) + .collect_vec(); + + // calling into the search module is not necessary, but reuses established patterns + list_all_splits(index_uids, &mut metastore).await? + }; let search_response_rest: Vec = - convert_to_es_cat_indices_response(index_uid_to_index_id, splits_metadata); + convert_to_es_cat_indices_response(&mut index_id_to_resp, splits_metadata); let search_response_rest = search_response_rest .into_iter() @@ -625,27 +627,24 @@ async fn es_scroll( } fn convert_to_es_cat_indices_response( - index_uid_to_index_id: HashMap, + index_id_to_resp: &mut HashMap, splits: Vec, ) -> Vec { - let mut per_index: HashMap = HashMap::new(); - for split_metadata in splits { - let index_id = index_uid_to_index_id - .get(&split_metadata.index_uid) + let index_stats_entry = index_id_to_resp + .get_mut(&split_metadata.index_uid) .unwrap_or_else(|| { panic!( - "index_uid {} not found in index_uid_to_index_id", + "index_id {} not found in index_id_to_resp", split_metadata.index_uid ) }); - let mut cat_index_entry: ElasticsearchCatIndexResponse = split_metadata.into(); - cat_index_entry.index = index_id.to_owned(); - - let index_stats_entry = per_index.entry(index_id.to_owned()).or_default(); + let cat_index_entry: ElasticsearchCatIndexResponse = split_metadata.into(); *index_stats_entry += cat_index_entry.clone(); } - let indices: Vec = per_index.values().cloned().collect(); + let mut indices: Vec = + index_id_to_resp.values().cloned().collect(); + indices.sort_by(|a, b| a.index.cmp(&b.index)); indices } diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml index b56f6c30053..ca59de616b3 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0021-cat-indices.yaml @@ -3,6 +3,8 @@ engines: - quickwit endpoint: "_cat/indices?format=json" expected: +- index: empty_index + docs.count: '0' - dataset.size: 222.8kb docs.count: '100' docs.deleted: '0' @@ -14,6 +16,10 @@ expected: status: open store.size: 271.8kb #uuid: gharchive:01HN2SDANHDN6WFAFNH7BBMQ8C +- index: otel-logs-v0_7 + docs.count: '0' +- index: otel-traces-v0_7 + docs.count: '0' --- method: [GET] engines: diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml index 4c675303cb6..1e29d7ffead 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.elasticsearch.yaml @@ -3,6 +3,20 @@ method: DELETE endpoint: gharchive status_code: null --- +# empty index +method: PUT +endpoint: empty_index +json: { + "mappings": { + "properties": { + "created_at": { + "type": "date", + "store": true + } + } + } +} +--- # Create index method: PUT endpoint: gharchive diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml index 811d8f4f393..bdc10d1249b 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml @@ -8,6 +8,20 @@ status_code: null method: POST api_root: http://localhost:7280/api/v1/ endpoint: indexes/ +json: + version: "0.7" + index_id: empty_index + doc_mapping: + field_mappings: + - name: created_at + type: datetime + fast: true +sleep_after: 3 +--- +# Create index +method: POST +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/ json: version: "0.7" index_id: gharchive diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.elasticsearch.yaml index 3bed8da1b05..d5e21e68f18 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.elasticsearch.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.elasticsearch.yaml @@ -1,3 +1,6 @@ # Delete possibly remaining index method: DELETE endpoint: gharchive +--- +method: DELETE +endpoint: empty_index diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml index 9450f8cf720..4b1a0e2a202 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_teardown.quickwit.yaml @@ -2,3 +2,8 @@ method: DELETE api_root: http://localhost:7280/api/v1/ endpoint: indexes/gharchive +--- +# Delete index +method: DELETE +api_root: http://localhost:7280/api/v1/ +endpoint: indexes/empty_index