quickwit-oss · fmassot · Jan 3, 2024 · Jan 1, 2024
diff --git a/config/quickwit.yaml b/config/quickwit.yaml
@@ -9,7 +9,7 @@
 #
 # Config file format version.
 #
-version: 0.6
+version: 0.7
 #
 # Node ID. Must be unique within a cluster. If not set, a random node ID is generated on each startup.
 #

diff --git a/config/tutorials/gh-archive/index-config-for-clickhouse.yaml b/config/tutorials/gh-archive/index-config-for-clickhouse.yaml
@@ -1,7 +1,7 @@
 #
 # Index config file for gh-archive dataset.
 #
-version: 0.6
+version: 0.7
 
 index_id: gh-archive
 

diff --git a/config/tutorials/gh-archive/index-config.yaml b/config/tutorials/gh-archive/index-config.yaml
@@ -1,7 +1,7 @@
 #
 # Index config file for gh-archive dataset.
 #
-version: 0.6
+version: 0.7
 
 index_id: gh-archive
 

diff --git a/config/tutorials/gh-archive/kafka-source.yaml b/config/tutorials/gh-archive/kafka-source.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 source_id: kafka-source
 source_type: kafka
 max_num_pipelines_per_indexer: 1

diff --git a/config/tutorials/gh-archive/kinesis-source.yaml b/config/tutorials/gh-archive/kinesis-source.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 source_id: kinesis-source
 source_type: kinesis
 params:

diff --git a/config/tutorials/grafana/docker-compose.yml b/config/tutorials/grafana/docker-compose.yml
@@ -0,0 +1,85 @@
+version: "3.9"
+
+networks:
+  default:
+    name: quickwit-grafana
+    # ipam:
+    #   config:
+    #   - subnet: 172.16.7.0/24
+    #     gateway: 172.16.7.1
+
+services:
+  quickwit:
+    image: quickwit/quickwit:${QUICKWIT_VERSION:-0.7.0}
+  grafana:
+    image: grafana/grafana-oss:${GRAFANA_VERSION:-9.4.7}
+    container_name: grafana
+    ports:
+      - "${MAP_HOST_GRAFANA:-127.0.0.1}:3000:3000"
+    environment:
+      GF_AUTH_DISABLE_LOGIN_FORM: "true"
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+    volumes:
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning
+
+  jaeger:
+    image: jaegertracing/all-in-one:${JAEGER_VERSION:-1.48.0}
+    container_name: jaeger
+    ports:
+      - "${MAP_HOST_JAEGER:-127.0.0.1}:16686:16686" # Frontend
+    profiles:
+      - jaeger
+      - monitoring
+
+  otel-collector:
+    image: otel/opentelemetry-collector:${OTEL_VERSION:-0.84.0}
+    container_name: otel-collector
+    ports:
+      - "${MAP_HOST_OTEL:-127.0.0.1}:1888:1888"   # pprof extension
+      - "${MAP_HOST_OTEL:-127.0.0.1}:8888:8888"   # Prometheus metrics exposed by the collector
+      - "${MAP_HOST_OTEL:-127.0.0.1}:8889:8889"   # Prometheus exporter metrics
+      - "${MAP_HOST_OTEL:-127.0.0.1}:13133:13133" # health_check extension
+      - "${MAP_HOST_OTEL:-127.0.0.1}:4317:4317"   # OTLP gRPC receiver
+      - "${MAP_HOST_OTEL:-127.0.0.1}:4318:4318"   # OTLP http receiver
+      - "${MAP_HOST_OTEL:-127.0.0.1}:55679:55679" # zpages extension
+    profiles:
+      - otel
+      - monitoring
+    volumes:
+      - ./monitoring/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    command: ["--config=/etc/otel-collector-config.yaml"]
+
+  prometheus:
+    image: prom/prometheus:${PROMETHEUS_VERSION:-v2.43.0}
+    container_name: prometheus
+    ports:
+      - "${MAP_HOST_PROMETHEUS:-127.0.0.1}:9090:9090"
+    profiles:
+      - prometheus
+      - monitoring
+    volumes:
+      - ./monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  gcp-pubsub-emulator:
+    # It is not an official docker image
+    # if we prefer we can build a docker from the official docker image (gcloud cli)
+    # and install the pubsub emulator https://cloud.google.com/pubsub/docs/emulator
+    image: thekevjames/gcloud-pubsub-emulator:${GCLOUD_EMULATOR:-455.0.0}
+    container_name: gcp-pubsub-emulator
+    ports:
+      - "${MAP_HOST_GCLOUD_EMULATOR:-127.0.0.1}:8681:8681"
+    environment:
+      # create a fake gcp project and a topic / subscription
+      - PUBSUB_PROJECT1=quickwit-emulator,emulator_topic:emulator_subscription
+    profiles:
+      - all
+      - gcp-pubsub
+
+volumes:
+  localstack_data:
+  postgres_data:
+  azurite_data:
diff --git a/config/tutorials/hdfs-logs/index-config-partitioned.yaml b/config/tutorials/hdfs-logs/index-config-partitioned.yaml
@@ -2,7 +2,7 @@
 # Index config file for hdfs-logs dataset with partitioning configured.
 #
 
-version: 0.6
+version: 0.7
 
 index_id: hdfs-logs-partitioned
 

diff --git a/config/tutorials/hdfs-logs/index-config-retention-policy.yaml b/config/tutorials/hdfs-logs/index-config-retention-policy.yaml
@@ -2,7 +2,7 @@
 # Index config file for hdfs-logs dataset with a retention policy configured.
 #
 
-version: 0.6
+version: 0.7
 
 index_id: hdfs-logs-retention-policy
 

diff --git a/config/tutorials/hdfs-logs/index-config.yaml b/config/tutorials/hdfs-logs/index-config.yaml
@@ -2,7 +2,7 @@
 # Index config file for hdfs-logs dataset.
 #
 
-version: 0.6
+version: 0.7
 
 index_id: hdfs-logs
 

diff --git a/config/tutorials/hdfs-logs/searcher-1.yaml b/config/tutorials/hdfs-logs/searcher-1.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 node_id: searcher-1
 listen_address: 127.0.0.1
 rest:

diff --git a/config/tutorials/hdfs-logs/searcher-2.yaml b/config/tutorials/hdfs-logs/searcher-2.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 node_id: searcher-2
 listen_address: 127.0.0.1
 rest:

diff --git a/config/tutorials/hdfs-logs/searcher-3.yaml b/config/tutorials/hdfs-logs/searcher-3.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 node_id: searcher-3
 listen_address: 127.0.0.1
 rest:

diff --git a/config/tutorials/otel-logs/index-config.yaml b/config/tutorials/otel-logs/index-config.yaml
@@ -3,7 +3,7 @@
 # Link: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md
 #
 
-version: 0.6
+version: 0.7
 
 index_id: otel-log-v0
 

diff --git a/config/tutorials/otel-traces/index-config.yaml b/config/tutorials/otel-traces/index-config.yaml
@@ -3,7 +3,7 @@
 # Link: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md
 #
 
-version: 0.6
+version: 0.7
 
 index_id: otel-trace-v0
 

diff --git a/config/tutorials/otel-traces/kafka-source.yaml b/config/tutorials/otel-traces/kafka-source.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 source_id: kafka-source
 source_type: kafka
 input_format: otlp_trace_proto

diff --git a/config/tutorials/stackoverflow/index-config.yaml b/config/tutorials/stackoverflow/index-config.yaml
@@ -1,7 +1,7 @@
 #
 # Index config file for stackoverflow dataset.
 #
-version: 0.6
+version: 0.7
 
 index_id: stackoverflow
 

diff --git a/config/tutorials/stackoverflow/pulsar-source.yaml b/config/tutorials/stackoverflow/pulsar-source.yaml
@@ -1,4 +1,4 @@
-version: 0.6
+version: 0.7
 source_id: pulsar-source
 source_type: pulsar
 params:

diff --git a/config/tutorials/wikipedia/index-config.yaml b/config/tutorials/wikipedia/index-config.yaml
@@ -2,7 +2,7 @@
 # Index config file for wikipedia dataset.
 #
 
-version: 0.6
+version: 0.7
 
 index_id: wikipedia
 

diff --git a/config/tutorials/wikipedia/multilang-index-config.yaml b/config/tutorials/wikipedia/multilang-index-config.yaml
@@ -2,7 +2,7 @@
 # Index config file for multilang wikipedia datasets.
 #
 
-version: 0.6
+version: 0.7
 
 index_id: multilang-wikipedia
 

diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md
@@ -21,7 +21,7 @@ The index configuration format is YAML. When a key is absent from the configurat
 Here is a complete example suited for the HDFS logs dataset:
 
 ```yaml
-version: 0.6 # File format version.
+version: 0.7 # File format version.
 
 index_id: "hdfs"
 
@@ -426,7 +426,7 @@ The configuration of `dynamic` mode can be set via the `dynamic_mapping` paramet
 `dynamic_mapping` offers the same configuration options as when configuring a `json` field. It defaults to:
 
 ```yaml
-version: 0.6
+version: 0.7
 index_id: my-dynamic-index
 doc_mapping:
   mode: dynamic
@@ -446,7 +446,7 @@ the root of the JSON object.
 For instance, in a entirely schemaless settings, a minimal index configuration could be:
 
 ```yaml
-version: 0.6
+version: 0.7
 index_id: my-dynamic-index
 doc_mapping:
     # If you have a timestamp field, it is important to tell quickwit about it.
@@ -532,7 +532,7 @@ Quickwit's default merge policy is the `stable_log` merge policy
 with the following parameters:
 
 ```yaml
-version: 0.6
+version: 0.7
 index_id: "hdfs"
 # ...
 indexing_settings:
@@ -561,7 +561,7 @@ of the number of merge operation a split should undergo.
 
 
 ```yaml
-version: 0.6
+version: 0.7
 index_id: "hdfs"
 # ...
 indexing_settings:
@@ -590,7 +590,7 @@ This setting is not recommended. Merges are necessary to reduce the number of sp
 :::
 
 ```yaml
-version: 0.6
+version: 0.7
 index_id: "hdfs"
 indexing_settings:
     merge_policy:
@@ -618,7 +618,7 @@ explicitly in the schema, or may refer to a field captured by the dynamic mode.
 This section describes how Quickwit manages data retention. In Quickwit, the retention policy manager drops data on a split basis as opposed to individually dropping documents. Splits are evaluated based on their `time_range` which is derived from the index timestamp field specified in the (`indexing_settings.timestamp_field`) settings. Using this setting, the retention policy will delete a split when `now() - split.time_range.end >= retention_policy.period`
 
 ```yaml
-version: 0.6
+version: 0.7
 index_id: hdfs
 # ...
 retention:

diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md
@@ -19,7 +19,7 @@ A commented example is available here: [quickwit.yaml](https://github.com/quickw
 
 | Property | Description | Env variable | Default value |
 | --- | --- | --- | --- |
-| `version` | Config file version. `0.6` is the only available value with a retro compatibility on `0.5` and `0.4`. | | |
+| `version` | Config file version. `0.7` is the only available value with a retro compatibility on `0.5` and `0.4`. | | |
 | `cluster_id` | Unique identifier of the cluster the node will be joining. Clusters sharing the same network should use distinct cluster IDs.| `QW_CLUSTER_ID` | `quickwit-default-cluster` |
 | `node_id` | Unique identifier of the node. It must be distinct from the node IDs of its cluster peers. Defaults to the instance's short hostname if not set. | `QW_NODE_ID` | short hostname |
 | `enabled_services` | Enabled services (control_plane, indexer, janitor, metastore, searcher) | `QW_ENABLED_SERVICES` | all services |
@@ -233,7 +233,7 @@ export QW_LISTEN_ADDRESS=0.0.0.0
 
 ```yaml
 # config.yaml
-version: 0.6
+version: 0.7
 cluster_id: quickwit-cluster
 node_id: my-unique-node-id
 listen_address: ${QW_LISTEN_ADDRESS}
@@ -244,7 +244,7 @@ rest:
 Will be interpreted by Quickwit as:
 
 ```yaml
-version: 0.6
+version: 0.7
 cluster_id: quickwit-cluster
 node_id: my-unique-node-id
 listen_address: 0.0.0.0

diff --git a/docs/configuration/source-config.md b/docs/configuration/source-config.md
@@ -80,7 +80,7 @@ Short max poll interval durations may cause a source to crash when back pressure
 
 ```bash
 cat << EOF > source-config.yaml
-version: 0.6
+version: 0.7
 source_id: my-kafka-source
 source_type: kafka
 max_num_pipelines_per_indexer: 1
@@ -124,7 +124,7 @@ If no region is specified, Quickwit will attempt to find one in multiple other l
 
 ```bash
 cat << EOF > source-config.yaml
-version: 0.6
+version: 0.7
 source_id: my-kinesis-source
 source_type: kinesis
 params:
@@ -153,7 +153,7 @@ The Pulsar source consumes `topics` using the client library [pulsar-rs](https:/
 
 ```bash
 cat << EOF > source-config.yaml
-version: 0.6
+version: 0.7
 source_id: my-pulsar-source
 source_type: pulsar
 params:

diff --git a/docs/distributed-tracing/otel-service.md b/docs/distributed-tracing/otel-service.md
@@ -27,7 +27,7 @@ Below is the doc mapping of the `otel-trace-v0` index:
 
 ```yaml
 
-version: 0.6
+version: 0.7
 
 index_id: otel-trace-v0
 

diff --git a/docs/get-started/installation.md b/docs/get-started/installation.md
@@ -43,13 +43,13 @@ Downloads `.tar.gz`:
 <!-- Bellow is the set of links to edit when a new version is released -->
 <RenderIf condition={useDocsVersion().version != 'current'}>
 
-version: 0.6.1 - [Release notes](https://github.com/quickwit-oss/quickwit/releases/tag/v0.6.1) - [Changelog](https://github.com/quickwit-oss/quickwit/blob/main/CHANGELOG.md)
+version: 0.7.0 - [Release notes](https://github.com/quickwit-oss/quickwit/releases/tag/v0.7.0) - [Changelog](https://github.com/quickwit-oss/quickwit/blob/main/CHANGELOG.md)
 License: [AGPL V3](https://github.com/quickwit-oss/quickwit/blob/main/LICENSE.md)
 Downloads `.tar.gz`:
-- [Linux ARM64](https://github.com/quickwit-oss/quickwit/releases/download/v0.6.1/quickwit-v0.6.1-aarch64-unknown-linux-gnu.tar.gz)
-- [Linux x86_64](https://github.com/quickwit-oss/quickwit/releases/download/v0.6.1/quickwit-v0.6.1-x86_64-unknown-linux-gnu.tar.gz)
-- [macOS aarch64](https://github.com/quickwit-oss/quickwit/releases/download/v0.6.1/quickwit-v0.6.1-aarch64-apple-darwin.tar.gz)
-- [macOS x86_64](https://github.com/quickwit-oss/quickwit/releases/download/v0.6.1/quickwit-v0.6.1-x86_64-apple-darwin.tar.gz)
+- [Linux ARM64](https://github.com/quickwit-oss/quickwit/releases/download/v0.7.0/quickwit-v0.7.0-aarch64-unknown-linux-gnu.tar.gz)
+- [Linux x86_64](https://github.com/quickwit-oss/quickwit/releases/download/v0.7.0/quickwit-v0.7.0-x86_64-unknown-linux-gnu.tar.gz)
+- [macOS aarch64](https://github.com/quickwit-oss/quickwit/releases/download/v0.7.0/quickwit-v0.7.0-aarch64-apple-darwin.tar.gz)
+- [macOS x86_64](https://github.com/quickwit-oss/quickwit/releases/download/v0.7.0/quickwit-v0.7.0-x86_64-apple-darwin.tar.gz)
 
 </RenderIf>
 

diff --git a/docs/get-started/quickstart.md b/docs/get-started/quickstart.md
@@ -87,7 +87,7 @@ And here is the complete config:
 #
 # Index config file for stackoverflow dataset.
 #
-version: 0.6
+version: 0.7
 
 index_id: stackoverflow
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,7 +27,7 @@ Below is the doc mapping of the `otel-trace-v0` index: @@
     ```yaml
-    version: 0.6
+    version: 0.7
     index_id: otel-trace-v0
@@ Expand Down @@