Remove v0.{4,5,6} backward compatibility files

quickwit-oss · Jun 6, 2024 · c2049ee · c2049ee
1 parent 0052b38
commit c2049ee
Show file tree

Hide file tree

Showing 27 changed files with 68 additions and 1,397 deletions.
diff --git a/quickwit/quickwit-config/src/index_config/serialize.rs b/quickwit/quickwit-config/src/index_config/serialize.rs
@@ -37,9 +37,6 @@ type IndexConfigForSerialization = IndexConfigV0_8;
 pub(crate) enum VersionedIndexConfig {
     #[serde(rename = "0.8")]
     // Retro compatibility
-    #[serde(alias = "0.4")]
-    #[serde(alias = "0.5")]
-    #[serde(alias = "0.6")]
     #[serde(alias = "0.7")]
     V0_8(IndexConfigV0_8),
 }

diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs
@@ -153,9 +153,6 @@ enum VersionedNodeConfig {
     #[serde(rename = "0.8")]
     // Retro compatibility.
     #[serde(alias = "0.7")]
-    #[serde(alias = "0.6")]
-    #[serde(alias = "0.5")]
-    #[serde(alias = "0.4")]
     V0_8(NodeConfigBuilder),
 }
 

diff --git a/quickwit/quickwit-config/src/source_config/serialize.rs b/quickwit/quickwit-config/src/source_config/serialize.rs
@@ -32,14 +32,11 @@ type SourceConfigForSerialization = SourceConfigV0_8;
 #[serde(deny_unknown_fields)]
 #[serde(tag = "version")]
 pub enum VersionedSourceConfig {
-    #[serde(rename = "0.7")]
-    // Retro compatibility.
-    #[serde(alias = "0.6")]
-    #[serde(alias = "0.5")]
-    #[serde(alias = "0.4")]
-    V0_7(SourceConfigV0_7),
     #[serde(rename = "0.8")]
     V0_8(SourceConfigV0_8),
+    // Retro compatibility.
+    #[serde(rename = "0.7")]
+    V0_7(SourceConfigV0_7),
 }
 
 impl From<VersionedSourceConfig> for SourceConfigForSerialization {

diff --git a/quickwit/quickwit-control-plane/src/control_plane.rs b/quickwit/quickwit-control-plane/src/control_plane.rs
@@ -2114,7 +2114,7 @@ mod tests {
             assert_eq!(source_configs[0].source_id, INGEST_V2_SOURCE_ID);
             assert_eq!(source_configs[1].source_id, CLI_SOURCE_ID);
 
-            let index_uid = IndexUid::from_parts("test-index-foo", 0);
+            let index_uid = IndexUid::for_test("test-index-foo", 0);
             let mut index_metadata = IndexMetadata::new_with_index_uid(index_uid, index_config);
 
             for source_config in source_configs {

diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs
@@ -806,11 +806,11 @@ mod tests {
     #[test]
     fn test_build_physical_indexing_plan_simple() {
         let source_1 = SourceUid {
-            index_uid: IndexUid::from_parts("index-1", 0),
+            index_uid: IndexUid::for_test("index-1", 0),
             source_id: "source1".to_string(),
         };
         let source_2 = SourceUid {
-            index_uid: IndexUid::from_parts("index-2", 0),
+            index_uid: IndexUid::for_test("index-2", 0),
             source_id: "source2".to_string(),
         };
         let sources = vec![
@@ -887,7 +887,7 @@ mod tests {
     prop_compose! {
       fn gen_kafka_source()
         (index_idx in 0usize..100usize, num_pipelines in 1usize..51usize) -> (IndexUid, SourceConfig) {
-          let index_uid = IndexUid::from_parts(&format!("index-id-{index_idx}"), 0 /* this is the index uid */);
+          let index_uid = IndexUid::for_test(&format!("index-id-{index_idx}"), 0 /* this is the index uid */);
           let source_id = quickwit_common::rand::append_random_suffix("kafka-source");
           (index_uid, SourceConfig {
               source_id,

diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs
@@ -727,7 +727,7 @@ mod tests {
 
     fn source_id() -> SourceUid {
         static COUNTER: AtomicUsize = AtomicUsize::new(0);
-        let index = IndexUid::from_parts("test_index", 0);
+        let index = IndexUid::for_test("test_index", 0);
         let source_id = COUNTER.fetch_add(1, Ordering::SeqCst);
         SourceUid {
             index_uid: index,

diff --git a/quickwit/quickwit-indexing/src/test_utils.rs b/quickwit/quickwit-indexing/src/test_utils.rs
@@ -256,7 +256,7 @@ pub struct MockSplitBuilder {
 impl MockSplitBuilder {
     pub fn new(split_id: &str) -> Self {
         Self {
-            split_metadata: mock_split_meta(split_id, &IndexUid::from_parts("test-index", 0)),
+            split_metadata: mock_split_meta(split_id, &IndexUid::for_test("test-index", 0)),
         }
     }
 

diff --git a/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs b/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs
@@ -493,7 +493,7 @@ mod tests {
     #[test]
     fn test_routing_table_entry_new() {
         let self_node_id: NodeId = "test-node-0".into();
-        let index_uid: IndexUid = IndexUid::from_parts("test-index", 0);
+        let index_uid = IndexUid::for_test("test-index", 0);
         let source_id: SourceId = "test-source".into();
         let table_entry = RoutingTableEntry::new(
             &self_node_id,
@@ -557,7 +557,7 @@ mod tests {
 
     #[test]
     fn test_routing_table_entry_has_open_shards() {
-        let index_uid: IndexUid = IndexUid::from_parts("test-index", 0);
+        let index_uid = IndexUid::for_test("test-index", 0);
         let source_id: SourceId = "test-source".into();
         let table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone());
 
@@ -653,7 +653,7 @@ mod tests {
 
     #[test]
     fn test_routing_table_entry_next_open_shard_round_robin() {
-        let index_uid: IndexUid = IndexUid::from_parts("test-index", 0);
+        let index_uid = IndexUid::for_test("test-index", 0);
         let source_id: SourceId = "test-source".into();
         let table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone());
         let ingester_pool = IngesterPool::default();
@@ -770,7 +770,7 @@ mod tests {
 
     #[test]
     fn test_routing_table_entry_insert_open_shards() {
-        let index_uid_0: IndexUid = IndexUid::from_parts("test-index", 0);
+        let index_uid_0 = IndexUid::for_test("test-index", 0);
         let source_id: SourceId = "test-source".into();
         let mut table_entry = RoutingTableEntry::empty(index_uid_0.clone(), source_id.clone());
 
@@ -847,7 +847,7 @@ mod tests {
         assert_eq!(table_entry.remote_shards[1].shard_state, ShardState::Closed);
 
         // Update index incarnation.
-        let index_uid_1: IndexUid = IndexUid::from_parts("test-index", 1);
+        let index_uid_1 = IndexUid::for_test("test-index", 1);
         table_entry.insert_open_shards(
             &local_node_id,
             &local_node_id,
@@ -879,7 +879,7 @@ mod tests {
 
     #[test]
     fn test_routing_table_entry_close_shards() {
-        let index_uid: IndexUid = IndexUid::from_parts("test-index", 0);
+        let index_uid = IndexUid::for_test("test-index", 0);
         let source_id: SourceId = "test-source".into();
 
         let mut table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone());
@@ -960,7 +960,7 @@ mod tests {
 
     #[test]
     fn test_routing_table_entry_delete_shards() {
-        let index_uid: IndexUid = IndexUid::from_parts("test-index", 0);
+        let index_uid = IndexUid::for_test("test-index", 0);
         let source_id: SourceId = "test-source".into();
 
         let mut table_entry = RoutingTableEntry::empty(index_uid.clone(), source_id.clone());

diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/mod.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/mod.rs
@@ -725,7 +725,6 @@ mod tests {
     use std::collections::BTreeSet;
 
     use quickwit_doc_mapper::tag_pruning::TagFilterAst;
-    use quickwit_doc_mapper::{BinaryFormat, FieldMappingType};
     use quickwit_proto::ingest::Shard;
     use quickwit_proto::metastore::ListShardsSubrequest;
     use quickwit_proto::types::{IndexUid, SourceId};
@@ -897,106 +896,4 @@ mod tests {
         assert!(!split_query_predicate(&&split_2, &query));
         assert!(!split_query_predicate(&&split_3, &query));
     }
-
-    #[test]
-    fn test_index_otel_bytes_fields_format_conversion() {
-        // TODO: remove after 0.8 release.
-        let index_json_str = r#"
-        {
-            "version": "0.6",
-            "splits": [],
-            "index": {
-                "version": "0.6",
-                "sources": [],
-                "index_uid": "otel-traces-v0_6:00000000000000000000000000",
-                "checkpoint": {
-                "kafka-source": {
-                    "00000000000000000000": "00000000000000000042"
-                }
-                },
-                "create_timestamp": 1789,
-                "index_config": {
-                    "version": "0.6",
-                    "index_id": "otel-traces-v0_6",
-                    "index_uri": "s3://otel-traces-v0_6",
-                    "doc_mapping": {
-                        "field_mappings": [
-                            {
-                                "name": "timestamp",
-                                "type": "datetime",
-                                "fast": true
-                            },
-                            {
-                                "name": "tenant_id",
-                                "type": "bytes",
-                                "fast": true,
-                                "input_format": "base64",
-                                "output_format": "base64"
-                            },
-                            {
-                                "name": "trace_id",
-                                "type": "bytes",
-                                "fast": true,
-                                "input_format": "base64",
-                                "output_format": "base64"
-                            },
-                            {
-                                "name": "span_id",
-                                "type": "bytes",
-                                "fast": true,
-                                "input_format": "base64",
-                                "output_format": "base64"
-                            }
-                        ],
-                        "tag_fields": [],
-                        "timestamp_field": "timestamp",
-                        "store_source": false
-                    }
-                }
-            }
-        }
-        "#;
-
-        let file_backed_index: FileBackedIndex = serde_json::from_str(index_json_str).unwrap();
-        let field_mapping = file_backed_index
-            .metadata
-            .index_config
-            .doc_mapping
-            .field_mappings;
-        assert_eq!(
-            field_mapping
-                .iter()
-                .filter(|field_mapping| field_mapping.name == "tenant_id")
-                .count(),
-            1
-        );
-        assert_eq!(
-            field_mapping
-                .iter()
-                .filter(|field_mapping| field_mapping.name == "trace_id")
-                .count(),
-            1
-        );
-        assert_eq!(
-            field_mapping
-                .iter()
-                .filter(|field_mapping| field_mapping.name == "span_id")
-                .count(),
-            1
-        );
-        for field_mapping in &field_mapping {
-            if field_mapping.name == "tenant_id" {
-                if let FieldMappingType::Bytes(bytes_options, _) = &field_mapping.mapping_type {
-                    assert_eq!(bytes_options.input_format, BinaryFormat::Base64);
-                    assert_eq!(bytes_options.output_format, BinaryFormat::Base64);
-                }
-            }
-            if field_mapping.name == "trace_id" || field_mapping.name == "span_id" {
-                if let FieldMappingType::Bytes(bytes_options, _) = &field_mapping.mapping_type {
-                    assert_eq!(bytes_options.input_format, BinaryFormat::Hex);
-                    assert_eq!(bytes_options.output_format, BinaryFormat::Hex);
-                }
-            }
-        }
-    }
 }
diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs
@@ -20,7 +20,6 @@
 use std::collections::HashMap;
 
 use itertools::Itertools;
-use quickwit_doc_mapper::{BinaryFormat, FieldMappingType};
 use quickwit_proto::ingest::Shard;
 use quickwit_proto::metastore::SourceType;
 use quickwit_proto::types::SourceId;
@@ -37,9 +36,6 @@ pub(crate) enum VersionedFileBackedIndex {
     #[serde(rename = "0.8")]
     // Retro compatibility.
     #[serde(alias = "0.7")]
-    #[serde(alias = "0.6")]
-    #[serde(alias = "0.5")]
-    #[serde(alias = "0.4")]
     V0_8(FileBackedIndexV0_8),
 }
 
@@ -106,35 +102,7 @@ impl From<FileBackedIndex> for FileBackedIndexV0_8 {
 }
 
 impl From<FileBackedIndexV0_8> for FileBackedIndex {
-    fn from(mut index: FileBackedIndexV0_8) -> Self {
-        // if the index is otel-traces-v0_6, convert set bytes fields input and output format to hex
-        // to be compatible with the v0_6 version.
-        // TODO: remove after 0.8 release.
-        if index.metadata.index_id() == "otel-traces-v0_6" {
-            index
-                .metadata
-                .index_config
-                .doc_mapping
-                .field_mappings
-                .iter_mut()
-                .filter(|field_mapping| {
-                    field_mapping.name == "trace_id" || field_mapping.name == "span_id"
-                })
-                .for_each(|field_mapping| {
-                    if let FieldMappingType::Bytes(bytes_options, _) =
-                        &mut field_mapping.mapping_type
-                    {
-                        bytes_options.input_format = BinaryFormat::Hex;
-                        bytes_options.output_format = BinaryFormat::Hex;
-                    }
-                });
-        }
-        // Override split index_id to support old SplitMetadata format.
-        for split in index.splits.iter_mut() {
-            if split.split_metadata.index_uid.is_empty() {
-                split.split_metadata.index_uid = index.metadata.index_uid.clone();
-            }
-        }
+    fn from(index: FileBackedIndexV0_8) -> Self {
         let mut per_source_shards: HashMap<SourceId, Shards> = index
             .shards
             .into_iter()

diff --git a/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs b/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs
@@ -31,7 +31,6 @@ use quickwit_proto::types::{IndexUid, Position, SourceId};
 use serde::{Deserialize, Serialize};
 use serialize::VersionedIndexMetadata;
 use time::OffsetDateTime;
-use ulid::Ulid;
 
 use crate::checkpoint::{IndexCheckpoint, PartitionId, SourceCheckpoint, SourceCheckpointDelta};
 
@@ -75,7 +74,7 @@ impl IndexMetadata {
     /// An incarnation id of `0` will be used to complete the index id into a index uuid.
     #[cfg(any(test, feature = "testsuite"))]
     pub fn for_test(index_id: &str, index_uri: &str) -> Self {
-        let index_uid = IndexUid::from_parts(index_id, 0);
+        let index_uid = IndexUid::for_test(index_id, 0);
         let mut index_metadata = IndexMetadata::new(IndexConfig::for_test(index_id, index_uri));
         index_metadata.index_uid = index_uid;
         index_metadata
@@ -161,6 +160,7 @@ impl IndexMetadata {
     }
 }
 
+#[cfg(any(test, feature = "testsuite"))]
 impl TestableForRegression for IndexMetadata {
     fn sample_for_regression() -> IndexMetadata {
         let mut source_checkpoint = SourceCheckpoint::default();
@@ -176,7 +176,7 @@ impl TestableForRegression for IndexMetadata {
         let checkpoint = IndexCheckpoint::from(per_source_checkpoint);
         let index_config = IndexConfig::sample_for_regression();
         let mut index_metadata = IndexMetadata {
-            index_uid: IndexUid::from_parts(&index_config.index_id, Ulid::nil()),
+            index_uid: IndexUid::for_test(&index_config.index_id, 0),
             index_config,
             checkpoint,
             create_timestamp: 1789,

diff --git a/quickwit/quickwit-metastore/src/metastore/index_metadata/serialize.rs b/quickwit/quickwit-metastore/src/metastore/index_metadata/serialize.rs
@@ -33,9 +33,6 @@ pub(crate) enum VersionedIndexMetadata {
     #[serde(rename = "0.8")]
     // Retro compatibility.
     #[serde(alias = "0.7")]
-    #[serde(alias = "0.6")]
-    #[serde(alias = "0.5")]
-    #[serde(alias = "0.4")]
     V0_8(IndexMetadataV0_8),
 }
 
@@ -73,8 +70,6 @@ impl From<IndexMetadata> for IndexMetadataV0_8 {
 #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)]
 pub(crate) struct IndexMetadataV0_8 {
     #[schema(value_type = String)]
-    // Defaults to nil for backward compatibility.
-    #[serde(default, alias = "index_id")]
     pub index_uid: IndexUid,
     #[schema(value_type = VersionedIndexConfig)]
     pub index_config: IndexConfig,
@@ -98,11 +93,7 @@ impl TryFrom<IndexMetadataV0_8> for IndexMetadata {
             sources.insert(source.source_id.clone(), source);
         }
         Ok(Self {
-            index_uid: if v0_8.index_uid.is_empty() {
-                IndexUid::from_parts(&v0_8.index_config.index_id, 0)
-            } else {
-                v0_8.index_uid
-            },
+            index_uid: v0_8.index_uid,
             index_config: v0_8.index_config,
             checkpoint: v0_8.checkpoint,
             create_timestamp: v0_8.create_timestamp,