add elastic search compatibility test

quickwit/quickwit-indexing/src/actors/packager.rs

@@ -334,7 +334,7 @@ fn create_packaged_split(
 /// Serializes the Split fields.
 ///
 /// `fields_metadata` has to be sorted.
-pub fn serialize_field_metadata(fields_metadata: &[FieldMetadata]) -> Vec<u8> {
+fn serialize_field_metadata(fields_metadata: &[FieldMetadata]) -> Vec<u8> {
     let fields = fields_metadata
         .iter()
         .map(field_metadata_to_list_field_serialized)

quickwit/quickwit-proto/src/search/mod.rs

@@ -264,15 +264,15 @@ pub fn serialize_split_fields(list_fields: ListFields) -> Vec<u8> {
 }
 
 /// Reads a fixed number of bytes into an array and returns the array.
-fn read_exact_array<R: Read, const N: usize>(reader: &mut R) -> io::Result<[u8; N]> {
+fn read_exact_array<const N: usize>(reader: &mut impl Read) -> io::Result<[u8; N]> {
     let mut buffer = [0u8; N];
     reader.read_exact(&mut buffer)?;
     Ok(buffer)
 }
 
 /// Reads the Split fields from a zstd compressed stream of bytes
 pub fn deserialize_split_fields<R: Read>(mut reader: R) -> io::Result<ListFields> {
-    let format_version = read_exact_array::<_, 1>(&mut reader)?[0];
+    let format_version = read_exact_array::<1>(&mut reader)?[0];
     if format_version != 2 {
         return Err(io::Error::new(
             io::ErrorKind::InvalidData,

quickwit/rest-api-tests/run_tests.py

@@ -65,7 +65,7 @@ def run_request_with_retry(run_req, expected_status_code=None, num_retries=10, w
         if try_number < num_retries:
             print("Retrying...")
             time.sleep(wait_time)
-    raise Exception("Wrong status code. Got %s, expected %s" % (r.status_code, expected_status_code))
+    raise Exception("Wrong status code. Got %s, expected %s, url %s" % (r.status_code, expected_status_code, run_req().url))
 
 
 def resolve_previous_result(c, previous_result):

quickwit/rest-api-tests/scenarii/es_field_capabilities/0001-field-capabilities.yaml

@@ -1,6 +1,8 @@
 # Test _field_caps API
 method: [GET]
-endpoint: _elastic/fieldcaps/_field_caps
+engines:
+  - quickwit
+endpoint: fieldcaps/_field_caps
 expected:
   indices:
   - fieldcaps
@@ -113,10 +115,12 @@ expected:
         - fieldcaps
 ---
 # Test fields parameter with `.dynamic` suffix
+engines:
+  - quickwit
 method: [GET]
 engines:
   - quickwit
-endpoint: _elastic/fieldcaps/_field_caps?fields=nested.response,nested.name
+endpoint: fieldcaps/_field_caps?fields=nested.response,nested.name
 expected:
   indices:
   - fieldcaps
@@ -146,7 +150,7 @@ expected:
 method: [GET]
 engines:
   - quickwit
-endpoint: _elastic/fieldcaps/_field_caps?fields=nest*
+endpoint: fieldcaps/_field_caps?fields=nest*
 expected:
   indices:
   - fieldcaps
@@ -176,7 +180,7 @@ expected:
 method: [GET]
 engines:
   - quickwit
-endpoint: _elastic/fieldcaps/_field_caps?fields=nest*
+endpoint: fieldcaps/_field_caps?fields=nest*
 expected:
   indices:
   - fieldcaps
@@ -206,7 +210,7 @@ expected:
 method: [GET]
 engines:
   - quickwit
-endpoint: _elastic/fieldcaps/_field_caps?fields=nested.*ponse
+endpoint: fieldcaps/_field_caps?fields=nested.*ponse
 expected:
   indices:
   - fieldcaps
@@ -218,5 +222,22 @@ expected:
         aggregatable: true
         indices:
         - fieldcaps
+---
+# Compare with elastic search
+method: [GET]
+engines:
+  - quickwit
+  - elasticsearch
+endpoint: fieldcaps/_field_caps?fields=nested.*ponse
+expected:
+  indices:
+  - fieldcaps
+  fields:
+    nested.response:
+      long:
+        metadata_field: false
+        searchable: true
+        aggregatable: true
+
 
 

quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.elasticsearch.yaml

@@ -0,0 +1,2 @@
+api_root: http://localhost:9200/
+

quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.quickwit.yaml

@@ -0,0 +1 @@
+api_root: "http://localhost:7280/api/v1/_elastic/"

quickwit/rest-api-tests/scenarii/es_field_capabilities/_ctx.yaml

@@ -1,5 +1,3 @@
-method: GET
-engines: ["quickwit"]
-api_root: "http://localhost:7280/api/v1/"
+method: [GET, POST]
 headers:
   Content-Type: application/json

quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.elasticsearch.yaml

@@ -0,0 +1,33 @@
+# Delete possibly remaining index
+method: DELETE
+endpoint: fieldcaps
+status_code: null
+---
+# Create index 1
+method: PUT
+endpoint: fieldcaps
+json: {
+  "mappings": {
+    "properties": {
+      "id": {
+        "type": "text",
+        "store": true,
+        "norms": false,
+        "index_options": "docs"
+      },
+    }
+  }
+}
+---
+# Ingest documents in fieldcaps
+method: POST
+endpoint: _bulk
+params:
+  refresh: "true"
+headers: {"Content-Type": "application/json"}
+ndjson:
+  - "index": { "_index": "fieldcaps" }
+  - {"id": 1, "name": "Fritz", "response": 30, "id": 5, "host": "192.168.0.1", "tags": ["nice", "cool"]}
+  - "index": { "_index": "fieldcaps" }
+  - {"id": 2, "nested": {"name": "Fritz", "response": 30}, "date": "2015-01-11T12:10:30Z", "host": "192.168.0.11", "tags": ["nice"]}
+

quickwit/rest-api-tests/scenarii/es_field_capabilities/_setup.quickwit.yaml

@@ -1,10 +1,12 @@
 # Delete possibly remaining index
 method: DELETE
+api_root: http://localhost:7280/api/v1/
 endpoint: indexes/fieldcaps
 status_code: null
 ---
 # Create index
 method: POST
+api_root: http://localhost:7280/api/v1/
 endpoint: indexes/
 json:
   version: "0.6"
@@ -24,6 +26,7 @@ json:
 ---
 # Ingest documents
 method: POST
+api_root: http://localhost:7280/api/v1/
 endpoint: fieldcaps/ingest
 num_retries: 10
 params:
@@ -34,6 +37,7 @@ ndjson:
 ---
 # Ingest documents split #2
 method: POST
+api_root: http://localhost:7280/api/v1/
 endpoint: fieldcaps/ingest
 params:
   commit: force
@@ -42,6 +46,7 @@ ndjson:
 ---
 # Ingest documents split #3
 method: POST
+api_root: http://localhost:7280/api/v1/
 endpoint: fieldcaps/ingest
 params:
   commit: force

quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.elasticsearch.yaml

@@ -0,0 +1,3 @@
+# # Delete index
+method: DELETE
+endpoint: fieldcaps

quickwit/rest-api-tests/scenarii/es_field_capabilities/_teardown.quickwit.yaml

@@ -1,3 +1,4 @@
 # # Delete index
 method: DELETE
+api_root: http://localhost:7280/api/v1/
 endpoint: indexes/fieldcaps