Replace all target configurations with new variant (#1015)

In #945, a new ("extended") target configuration was introduced: ```yaml target: - a - b ``` instead of `target: [a, b]`. We still maintain compatibility with the old target configuration. However, we should educate users (in our docs) to use the new target configuration. This PR replaces all, internal and external (docs), uses of old target configuration with the new ("extended") target configuration. A new test (`TestTargetLegacyVariant`) is added to make sure we still maintain compatibility with the old way of declaring targets.
QuesmaOrg · Nov 20, 2024 · 6b5c479 · 6b5c479
1 parent d59a9f2
commit 6b5c479
Show file tree

Hide file tree

Showing 24 changed files with 400 additions and 158 deletions.
diff --git a/ci/it/configs/quesma-now-reads-clickhouse-tables.yml.template b/ci/it/configs/quesma-now-reads-clickhouse-tables.yml.template
@@ -25,9 +25,11 @@ processors:
     config:
       indexes:
         test_table:
-          target: [ my-clickhouse-instance]
+          target:
+            - my-clickhouse-instance
         "*":
-          target: [my-minimal-elasticsearch]
+          target:
+            - my-minimal-elasticsearch
 pipelines:
   - name: my-elasticsearch-proxy-read
     frontendConnectors: [ elastic-query ]

diff --git a/ci/it/configs/quesma-wildcard-clickhouse.yml.template b/ci/it/configs/quesma-wildcard-clickhouse.yml.template
@@ -23,15 +23,16 @@ processors:
   - name: my-query-processor
     type: quesma-v1-processor-query
     config:
-      indexes:
+     indexes:
         "*":
-          target: [ my-clickhouse-instance ]
+          target:
+            - my-clickhouse-instance
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
       indexes:
         "*":
-          target: [ my-clickhouse-instance ]
+          target: [ my-clickhouse-instance ] # deliberately using legacy way of listing targets here (for better coverage)
 pipelines:
   - name: my-elasticsearch-proxy-read
     frontendConnectors: [ elastic-query ]

diff --git a/ci/it/configs/quesma-wildcard-disabled.yml.template b/ci/it/configs/quesma-wildcard-disabled.yml.template
@@ -25,25 +25,27 @@ processors:
     config:
       indexes:
         "explicitly_disabled1":
-          target: [ ]
+          target:
         "explicitly_disabled3":
-          target: [ ]
+          target:
         "query_enabled":
-          target: [ my-clickhouse-instance ]
+          target:
+            - my-clickhouse-instance
         "*":
-          target: [ ]
+          target:
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
       indexes:
         "explicitly_disabled2":
-          target: [ ]
+          target:
         "explicitly_disabled3":
-          target: [ ]
+          target:
         "ingest_enabled":
-          target: [ my-clickhouse-instance ]
+          target:
+            - my-clickhouse-instance
         "*":
-          target: [ ]
+          target:
 pipelines:
   - name: my-elasticsearch-proxy-read
     frontendConnectors: [ elastic-query ]
@@ -53,4 +55,3 @@ pipelines:
     frontendConnectors: [ elastic-ingest ]
     processors: [ my-ingest-processor ]
     backendConnectors: [ my-minimal-elasticsearch, my-clickhouse-instance ]
-
diff --git a/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template b/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template
@@ -25,14 +25,20 @@ processors:
     config:
       indexes:
         logs-1:
-          target: [ e ]
+          target:
+            - e
         logs-2:
-          target: [ c ]
+          target:
+            - c
         logs-3:
-          target: [ c, e ]
+          target:
+            - c
+            - e
         ## WARNING `logs-dual-query` (with two dashes) falls under default index pattern for logs in Elasticsearch and results in not index, but datastream creation
         logs-dual_query:
-          target: [ c, e ]
+          target:
+            - c
+            - e
         logs-4:
           target:
             - c:
@@ -42,27 +48,35 @@ processors:
             - c:
                 useCommonTable: true
         "*":
-          target: [ e ]
+          target:
+            - e
   - name: IP
     type: quesma-v1-processor-ingest
     config:
       indexes:
         logs-1:
-          target: [ e ]
+          target:
+            - e
         logs-2:
-          target: [ c ]
+          target:
+            - c
         logs-3:
-          target: [ c, e ]
+          target:
+            - c
+            - e
         logs-dual_query:
-          target: [ c, e ]
+          target:
+            - c
+            - e
         logs-4:
           target:
             - c:
                 useCommonTable: true
         logs-5:
           target:
         "*":
-          target: [ e ]
+          target:
+            - e
 
 pipelines:
   - name: my-elasticsearch-proxy-read
@@ -72,4 +86,4 @@ pipelines:
   - name: my-elasticsearch-proxy-write
     frontendConnectors: [ elastic-ingest ]
     processors: [ IP ]
-    backendConnectors: [ e, c ]
+    backendConnectors: [ e, c ]
diff --git a/ci/it/configs/quesma-with-two-pipelines.yml.template b/ci/it/configs/quesma-with-two-pipelines.yml.template
@@ -25,23 +25,29 @@ processors:
     config:
       indexes:
         siem:
-          target: [my-hydrolix-instance]
+          target:
+            - my-hydrolix-instance
         logs:
-          target: [my-hydrolix-instance]
+          target:
+            - my-hydrolix-instance
         test_index:
-          target: [my-minimal-elasticsearch]
+          target:
+            - my-minimal-elasticsearch
         "*":
-          target: [ my-minimal-elasticsearch ]
+          target:
+            - my-minimal-elasticsearch
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
       indexes:
         test_index:
-          target: [ my-minimal-elasticsearch ]
+          target:
+            - my-minimal-elasticsearch
         logs_disabled:
-          target: [  ]
+          target:
         "*":
-          target: [ my-minimal-elasticsearch ]
+          target:
+            - my-minimal-elasticsearch
 pipelines:
   - name: my-elasticsearch-proxy-read
     frontendConnectors: [ elastic-query ]
@@ -51,4 +57,3 @@ pipelines:
     frontendConnectors: [ elastic-ingest ]
     processors: [ my-ingest-processor ]
     backendConnectors: [ my-minimal-elasticsearch, my-hydrolix-instance ]
-
diff --git a/ci/quesma/config/ci-config.yaml b/ci/quesma/config/ci-config.yaml
@@ -24,41 +24,53 @@ processors:
     config:
       indexes:
         kibana_sample_data_ecommerce:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         kibana_sample_data_flights:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         kibana_sample_data_logs:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         logs-generic-default:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
           schemaOverrides:
             fields:
               "message":
                 type: text
         windows_logs:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         "*":
-          target: [ my-minimal-elasticsearch ]
+          target:
+            - my-minimal-elasticsearch
   - name: p2
     type: quesma-v1-processor-ingest
     config:
       indexes:
         kibana_sample_data_ecommerce:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         kibana_sample_data_flights:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         kibana_sample_data_logs:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         logs-generic-default:
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
           schemaOverrides:
             fields:
               "message":
                 type: text
         windows_logs:   # Used for EQL e2e tests
-          target: [ my-clickhouse-data-source ]
+          target:
+            - my-clickhouse-data-source
         "*":
-          target: [ my-minimal-elasticsearch ]
+          target:
+            - my-minimal-elasticsearch
 
 pipelines:
   - name: p-query

diff --git a/docs/public/docs/compatibility-report.md b/docs/public/docs/compatibility-report.md
@@ -17,15 +17,19 @@ processors:
     config:
       indexes:
         kibana_sample_data_ecommerce:
-          target: [ backend-elastic, backend-clickhouse ]
+          target:
+            - backend-elastic
+            - backend-clickhouse
         ab_testing_logs:
-          target: [ backend-clickhouse ]
+          target:
+            - backend-clickhouse
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
       indexes:
         ab_testing_logs:
-          target: [ backend-clickhouse ]
+          target:
+            - backend-clickhouse
 ```
 
 This configuration turns on Compatibility Report for `kibana_sample_data_ecommerce` index. The `ab_testing_logs` is an internal Quesma index which is required in the configuration for Compatibility Report to work properly.

diff --git a/docs/public/docs/config-primer.md b/docs/public/docs/config-primer.md
@@ -120,15 +120,17 @@ processors:
     config:
       indexes:
         kibana_sample_data_ecommerce:
-          target: [ backend-clickhouse ]
+          target:
+            - backend-clickhouse
           schemaOverrides:
             fields:
               "geoip.location":
                 type: geo_point
               "products.product_name":
                 type: text
         "*":
-          target: [ backend-elastic ]
+          target:
+            - backend-elastic
 ```
 
 To get more specific information on processor configuration, please refer to [Processor configuration](#processor-configuration) section. 
@@ -168,15 +170,18 @@ processors:
     config:
       indexes:
         kibana_sample_data_logs:
-          target: [ backend-elasticsearch ]
+          target:
+            - backend-elasticsearch
         kibana_sample_data_ecommerce:
-          target: [ backend-clickhouse ]
+          target:
+            - backend-clickhouse
           schemaOverrides:
             fields:
               "geoip.location":
                 type: geo_point
         "*": # Always required
-          target: [ backend-elasticsearch ]
+          target:
+            - backend-elasticsearch
 ```
 
 ### Index configuration
@@ -187,11 +192,13 @@ The configuration for an index consists of the following configuration options:
 - `target` (required): a list of backend connectors that will handle the request. For example the following configuration in the ingest processor:
    ```yaml
      my_index:
-       target: [ backend-elasticsearch, backend-clickhouse ]
+       target:
+         - backend-elasticsearch
+         - backend-clickhouse
    ```
    will dual write ingest requests to `my_index` to both ElasticSearch and ClickHouse.
    Note that ElasticSearch/OpenSearch is the only supported backend for the `*` entry.
-   If no targets are provided (example: `target: []`) in the configuration of an index in the ingest processor, ingest for that index will be disabled and incoming data will be dropped.
+   If no targets are provided (empty `target` list) in the configuration of an index in the ingest processor, ingest for that index will be disabled and incoming data will be dropped.
    For the query processor by specifing multiple targets, [Compatibility report](/compatibility-report.md) will be enabled. See [Compatibility report documentation](/compatibility-report.md) for more details.
 
    Some backend connectors have additional attributes which may be used. For example the following configuration sets `useCommonTable` for `backend-clickhouse` target:
@@ -207,7 +214,8 @@ The configuration for an index consists of the following configuration options:
 - `schemaOverrides` (optional): manual overrides of schema information for an index. Quesma infers schema for an index based on the data ingested and the schema information fetched from ClickHouse. `schemaOverrides` allows you to override this inferred schema with for some fields. For example the following configuration:
     ```yaml
       my_index:
-        target: [ backend-clickhouse ]
+        target:
+          - backend-clickhouse
         schemaOverrides:
           "product_name":
             type: "text"

diff --git a/docs/public/docs/example-2-0-clickhouse-specific.md b/docs/public/docs/example-2-0-clickhouse-specific.md
@@ -72,21 +72,27 @@ flowchart LR
         config:
           indexes:      # `siem` and `logs` are just examples, 
             siem:       # make sure to replace them with your actual table names in your ClickHouse instance
-              target: [ clickhouse-instance ]
+              target:
+                - clickhouse-instance
             logs:
-              target: [ clickhouse-instance ]
+              target:
+                - clickhouse-instance
             '*':        # DO NOT remove, always required
-              target: [ minimal-elasticsearch ]
+              target:
+                - minimal-elasticsearch
       - name: ingest-processor
         type: quesma-v1-processor-ingest
         config:
           indexes:      # `siem` and `logs` are just examples, 
             siem:       # make sure to replace them with your actual table or index names in your ClickHouse instance
-              target: [ clickhouse-instance ]
+              target:
+                - clickhouse-instance
             logs:
-              target: [ clickhouse-instance ]
+              target:
+                - clickhouse-instance
             '*':        # DO NOT remove, always required
-              target: [ minimal-elasticsearch ]  
+              target:
+                - minimal-elasticsearch
     pipelines:
       - name: elasticsearch-proxy-read
         frontendConnectors: [ elastic-query ]