From f163317403cd6be92240c4ae64e00e396c9798bd Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Mon, 18 Dec 2023 15:51:06 +0800 Subject: [PATCH] add elastic search compatibility test --- .../quickwit-indexing/src/actors/packager.rs | 2 +- quickwit/quickwit-proto/src/search/mod.rs | 4 +-- quickwit/rest-api-tests/run_tests.py | 2 +- .../0001-field-capabilities.yaml | 31 ++++++++++++++--- .../_ctx.elasticsearch.yaml | 2 ++ .../es_field_capabilities/_ctx.quickwit.yaml | 1 + .../scenarii/es_field_capabilities/_ctx.yaml | 4 +-- .../_setup.elasticsearch.yaml | 33 +++++++++++++++++++ .../_setup.quickwit.yaml | 5 +++ .../_teardown.elasticsearch.yaml | 3 ++ .../_teardown.quickwit.yaml | 1 + 11 files changed, 76 insertions(+), 12 deletions(-) create mode 100644 quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.elasticsearch.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.quickwit.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.elasticsearch.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.elasticsearch.yaml diff --git a/quickwit/quickwit-indexing/src/actors/packager.rs b/quickwit/quickwit-indexing/src/actors/packager.rs index 0acc6d0de42..69e46d52949 100644 --- a/quickwit/quickwit-indexing/src/actors/packager.rs +++ b/quickwit/quickwit-indexing/src/actors/packager.rs @@ -334,7 +334,7 @@ fn create_packaged_split( /// Serializes the Split fields. /// /// `fields_metadata` has to be sorted. -pub fn serialize_field_metadata(fields_metadata: &[FieldMetadata]) -> Vec { +fn serialize_field_metadata(fields_metadata: &[FieldMetadata]) -> Vec { let fields = fields_metadata .iter() .map(field_metadata_to_list_field_serialized) diff --git a/quickwit/quickwit-proto/src/search/mod.rs b/quickwit/quickwit-proto/src/search/mod.rs index 682c5041bd1..3f3ba89855a 100644 --- a/quickwit/quickwit-proto/src/search/mod.rs +++ b/quickwit/quickwit-proto/src/search/mod.rs @@ -264,7 +264,7 @@ pub fn serialize_split_fields(list_fields: ListFields) -> Vec { } /// Reads a fixed number of bytes into an array and returns the array. -fn read_exact_array(reader: &mut R) -> io::Result<[u8; N]> { +fn read_exact_array(reader: &mut impl Read) -> io::Result<[u8; N]> { let mut buffer = [0u8; N]; reader.read_exact(&mut buffer)?; Ok(buffer) @@ -272,7 +272,7 @@ fn read_exact_array(reader: &mut R) -> io::Result<[u8; /// Reads the Split fields from a zstd compressed stream of bytes pub fn deserialize_split_fields(mut reader: R) -> io::Result { - let format_version = read_exact_array::<_, 1>(&mut reader)?[0]; + let format_version = read_exact_array::<1>(&mut reader)?[0]; if format_version != 2 { return Err(io::Error::new( io::ErrorKind::InvalidData, diff --git a/quickwit/rest-api-tests/run_tests.py b/quickwit/rest-api-tests/run_tests.py index 471000c6a46..be4195baf4a 100755 --- a/quickwit/rest-api-tests/run_tests.py +++ b/quickwit/rest-api-tests/run_tests.py @@ -65,7 +65,7 @@ def run_request_with_retry(run_req, expected_status_code=None, num_retries=10, w if try_number < num_retries: print("Retrying...") time.sleep(wait_time) - raise Exception("Wrong status code. Got %s, expected %s" % (r.status_code, expected_status_code)) + raise Exception("Wrong status code. Got %s, expected %s, url %s" % (r.status_code, expected_status_code, run_req().url)) def resolve_previous_result(c, previous_result): diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml index 40c2f633c6d..b81bc230ac3 100644 --- a/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml @@ -1,6 +1,8 @@ # Test _field_caps API method: [GET] -endpoint: _elastic/fieldcaps/_field_caps +engines: + - quickwit +endpoint: fieldcaps/_field_caps expected: indices: - fieldcaps @@ -113,10 +115,12 @@ expected: - fieldcaps --- # Test fields parameter with `.dynamic` suffix +engines: + - quickwit method: [GET] engines: - quickwit -endpoint: _elastic/fieldcaps/_field_caps?fields=nested.response,nested.name +endpoint: fieldcaps/_field_caps?fields=nested.response,nested.name expected: indices: - fieldcaps @@ -146,7 +150,7 @@ expected: method: [GET] engines: - quickwit -endpoint: _elastic/fieldcaps/_field_caps?fields=nest* +endpoint: fieldcaps/_field_caps?fields=nest* expected: indices: - fieldcaps @@ -176,7 +180,7 @@ expected: method: [GET] engines: - quickwit -endpoint: _elastic/fieldcaps/_field_caps?fields=nest* +endpoint: fieldcaps/_field_caps?fields=nest* expected: indices: - fieldcaps @@ -206,7 +210,7 @@ expected: method: [GET] engines: - quickwit -endpoint: _elastic/fieldcaps/_field_caps?fields=nested.*ponse +endpoint: fieldcaps/_field_caps?fields=nested.*ponse expected: indices: - fieldcaps @@ -218,5 +222,22 @@ expected: aggregatable: true indices: - fieldcaps +--- +# Compare with elastic search +method: [GET] +engines: + - quickwit + - elasticsearch +endpoint: fieldcaps/_field_caps?fields=nested.*ponse +expected: + indices: + - fieldcaps + fields: + nested.response: + long: + metadata_field: false + searchable: true + aggregatable: true + diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.elasticsearch.yaml new file mode 100644 index 00000000000..d83aa5b5e8d --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.elasticsearch.yaml @@ -0,0 +1,2 @@ +api_root: http://localhost:9200/ + diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.quickwit.yaml new file mode 100644 index 00000000000..e0028a1af4b --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.quickwit.yaml @@ -0,0 +1 @@ +api_root: "http://localhost:7280/api/v1/_elastic/" diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.yaml index f507346eae6..09cb0081b4f 100644 --- a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.yaml +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.yaml @@ -1,5 +1,3 @@ -method: GET -engines: ["quickwit"] -api_root: "http://localhost:7280/api/v1/" +method: [GET, POST] headers: Content-Type: application/json diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.elasticsearch.yaml new file mode 100644 index 00000000000..6a19e24a741 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.elasticsearch.yaml @@ -0,0 +1,33 @@ +# Delete possibly remaining index +method: DELETE +endpoint: fieldcaps +status_code: null +--- +# Create index 1 +method: PUT +endpoint: fieldcaps +json: { + "mappings": { + "properties": { + "id": { + "type": "text", + "store": true, + "norms": false, + "index_options": "docs" + }, + } + } +} +--- +# Ingest documents in fieldcaps +method: POST +endpoint: _bulk +params: + refresh: "true" +headers: {"Content-Type": "application/json"} +ndjson: + - "index": { "_index": "fieldcaps" } + - {"id": 1, "name": "Fritz", "response": 30, "id": 5, "host": "192.168.0.1", "tags": ["nice", "cool"]} + - "index": { "_index": "fieldcaps" } + - {"id": 2, "nested": {"name": "Fritz", "response": 30}, "date": "2015-01-11T12:10:30Z", "host": "192.168.0.11", "tags": ["nice"]} + diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml index 857551113cb..5de65633d1b 100644 --- a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml @@ -1,10 +1,12 @@ # Delete possibly remaining index method: DELETE +api_root: http://localhost:7280/api/v1/ endpoint: indexes/fieldcaps status_code: null --- # Create index method: POST +api_root: http://localhost:7280/api/v1/ endpoint: indexes/ json: version: "0.6" @@ -24,6 +26,7 @@ json: --- # Ingest documents method: POST +api_root: http://localhost:7280/api/v1/ endpoint: fieldcaps/ingest num_retries: 10 params: @@ -34,6 +37,7 @@ ndjson: --- # Ingest documents split #2 method: POST +api_root: http://localhost:7280/api/v1/ endpoint: fieldcaps/ingest params: commit: force @@ -42,6 +46,7 @@ ndjson: --- # Ingest documents split #3 method: POST +api_root: http://localhost:7280/api/v1/ endpoint: fieldcaps/ingest params: commit: force diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.elasticsearch.yaml new file mode 100644 index 00000000000..9d3c1723eed --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.elasticsearch.yaml @@ -0,0 +1,3 @@ +# # Delete index +method: DELETE +endpoint: fieldcaps diff --git a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.quickwit.yaml index af46057d19b..66330c067ad 100644 --- a/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.quickwit.yaml @@ -1,3 +1,4 @@ # # Delete index method: DELETE +api_root: http://localhost:7280/api/v1/ endpoint: indexes/fieldcaps