chore: add demo for vector

vector-ingestion/example.toml

@@ -0,0 +1,44 @@
+[sources.host_metrics]
+type = "host_metrics"
+collectors = ["cpu", "load", "memory"]
+# namespace = "metrics"
+scrape_interval_secs = 5
+
+[sources.apache_logs]
+type = "demo_logs"
+count = 100
+format = "apache_common"
+interval = 1
+lines = ["line1"]
+
+[sinks.metrics]
+type = "greptimedb_metrics"
+inputs = ["host_metrics"]
+endpoint = "${GT_HOST}:${GT_GRPC_PORT:-5001}"
+dbname = "${GT_DB_NAME}"
+username = "${GT_USERNAME}"
+password = "${GT_PASSWORD}"
+grpc_compression = "gzip"
+###tls = {}
+
+[sinks.logs]
+type = "greptimedb_logs"
+inputs = ["apache_logs"]
+compression = "gzip"
+dbname = "${GT_DB_NAME}"
+endpoint = "${GT_SCHEMA-:https}://${GT_HOST}:${GT_HTTP_PORT:-80}"
+username = "${GT_USERNAME}"
+password = "${GT_PASSWORD}"
+pipeline_name = "greptime_identity"
+table = "demo_logs"
+
+[sinks.structured_logs]
+type = "greptimedb_logs"
+inputs = ["apache_logs"]
+compression = "gzip"
+dbname = "${GT_DB_NAME}"
+endpoint = "${GT_SCHEMA-:https}://${GT_HOST}:${GT_HTTP_PORT:-80}"
+username = "${GT_USERNAME}"
+password = "${GT_PASSWORD}"
+pipeline_name = "apache_common_pipeline"
+table = "demo_structured_logs"

vector-ingestion/pipeline.yaml

@@ -0,0 +1,40 @@
+#145.251.205.39 - devankoshal [29/Oct/2024:17:51:29 +0800] "DELETE /booper/bopper/mooper/mopper HTTP/1.1" 304 43420
+processors:
+  - dissect:
+      fields:
+        - message
+      patterns:
+        - '%{ip_address} - %{user} [%{request_time}] "%{http_method} %{request_path} HTTP/%{http_version}" %{status_code} %{response_size}'
+      ignore_missing: true
+  - date:
+      fields:
+        - request_time
+      formats:
+        # 29/Oct/2024:17:51:29 +0800
+        - "%d/%b/%Y:%H:%M:%S %z"
+
+transform:
+  - fields:
+      - ip_address
+      - http_method
+      - http_version
+    type: string
+    index: tag
+  - fields:
+      - status_code
+    type: int32
+    index: tag
+  - fields:
+      - request_path
+    type: string
+    index: fulltext
+  - fields:
+      - user
+    type: string
+  - fields:
+      - response_size
+    type: int32
+  - fields:
+      - request_time
+    type: time
+    index: timestamp

vector-ingestion/start.env

@@ -0,0 +1,10 @@
+GT_SCHEMA=http
+GT_HOST=localhost
+GT_DB_NAME=public
+GT_USERNAME=
+GT_PASSWORD=
+GT_GTPC_TLS=false
+GT_GRPC_PORT=4001
+GT_HTTP_PORT=4000
+
+

vector-ingestion/start.sh

@@ -0,0 +1,70 @@
+[ ! -f start.env ] || export $(grep -v '^#' start.env | xargs)
+
+GT_SCHEMA=${GT_SCHEMA-:https}
+
+if [ -z "$GT_SCHEMA" ]; then
+  echo "GT_SCHEMA is not set"
+  exit 1
+fi
+
+if [ -z "$GT_HOST" ]; then
+  echo "GT_HOST is not set"
+  exit 1
+fi
+
+if [ -z "$GT_DB_NAME" ]; then
+  echo "GT_DB_NAME is not set"
+  exit 1
+fi
+
+function create_pipeline {
+## if GT_USERNAME and GT_PASSWORD are not set, then disable basic auth
+
+if [ -z "$GT_USERNAME" ] || [ -z "$GT_PASSWORD" ];
+then
+  echo "GT_USERNAME or GT_PASSWORD is not set, disabling basic auth"
+  curl -X "POST" "$GT_SCHEMA://$GT_HOST:$GT_HTTP_PORT/v1/events/pipelines/apache_common_pipeline?db=$GT_DB_NAME" \
+    -F "[email protected]"
+else
+  echo "GT_USERNAME and GT_PASSWORD are set, enabling basic"
+  curl -X "POST" "$GT_SCHEMA://$GT_HOST:$GT_HTTP_PORT/v1/events/pipelines/apache_common_pipeline?db=$GT_DB_NAME" \
+     -u "$GT_USERNAME:$GT_PASSWORD" \
+     -F "[email protected]"
+fi 
+}
+
+function run_demo {
+
+## if GT_GRPC_TLS is true set example.toml to use tls
+if [ "$GT_GRPC_TLS" = "true" ];
+then
+  echo "GT_GRPC_TLS is true, setting example.toml grpc to use tls"
+  sed -i 's/#tls = {}/tls = {}/g' example.toml
+else
+  echo "GT_GRPC_TLS is not set or false, setting example.toml grpc to not use tls"
+  sed -i 's/tls = {}/#tls = {}/g' example.toml
+fi
+
+podman run \
+  --env-file start.env \
+  --rm \
+  -v $PWD/example.toml:/etc/vector/vector.toml:ro \
+  --name vector \
+  --network host \
+  timberio/vector:0.42.0-debian  --config-toml /etc/vector/vector.toml
+
+}
+
+## two subcommands are supported, create_pipeline and run_demo
+case "$1" in
+  create_pipeline)
+    create_pipeline
+    ;;
+  run_demo)
+    run_demo
+    ;;
+  *)
+    echo "Usage: $0 {create_pipeline|run_demo}"
+    exit 1
+    ;;
+esac