feat: initial commit of nginx log and metrics demo

nginx-log-metrics/build_vector/Dockerfile

@@ -0,0 +1,29 @@
+# Use Rust as the base image
+FROM rust:1.79 as builder
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y cmake pkg-config libssl-dev protobuf-compiler libsasl2-dev
+
+# Clone the Vector source code
+RUN git clone https://github.com/shuiyisong/vector.git /vector
+WORKDIR /vector
+
+# Switch to your custom branch
+RUN git checkout chore/greptime_log_ingester
+
+# Build Vector
+RUN cargo build --release --no-default-features --features=default-greptimedb-custom
+
+# Use a minimal base image for the final image
+FROM debian:bookworm-slim
+
+# RUN apt-get update && apt-get install -y libc-bin=2.13-38+deb7u4 libc6=2.13-38+deb7u4
+
+# Copy the Vector binary from the builder stage
+COPY --from=builder /vector/target/release/vector /usr/local/bin/vector
+
+# Set up the config directory
+ENV VECTOR_CONFIG_LOCATION "/etc/vector/vector.toml"
+
+# Define the entry point
+CMD vector -c $VECTOR_CONFIG_LOCATION

nginx-log-metrics/config_data/prometheus-greptimedb.yml

@@ -0,0 +1,20 @@
+# my global config
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "node"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ["nginx_exporter:9113"]
+
+remote_write:
+  - url: http://greptimedb:4000/v1/prometheus/write?db=public

nginx-log-metrics/config_data/vector.toml

@@ -0,0 +1,21 @@
+[sources.file_source]
+type = "file"
+data_dir = "/logs"
+include = [ "/logs/access.log" ]
+# 512 mb
+max_read_bytes = 536870912
+
+[transforms.greptime_trans]
+type = "remap"
+inputs = ["file_source"]
+  source = '''
+  . = parse_json!(.message)
+'''
+
+[sinks.sink_greptime_logs]
+type = "greptimedb_logs"
+table = "demo_table"
+pipeline_name = "demo_pipeline"
+compression = "gzip"
+inputs = [ "greptime_trans" ]
+endpoint = "http://greptimedb:4000"

nginx-log-metrics/docker-compose.yml

@@ -0,0 +1,107 @@
+services:
+  greptimedb:
+    image: docker.io/greptime/greptimedb:v0.9.0-nightly-20240701
+    container_name: greptimedb
+    command: standalone start --http-addr=0.0.0.0:4000 --rpc-addr=0.0.0.0:4001 --mysql-addr=0.0.0.0:4002 --postgres-addr 0.0.0.0:4003
+    ports:
+      - 4000:4000
+      - 4001:4001
+      - 4002:4002
+      - 4003:4003
+    networks:
+      - demo-network
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://127.0.0.1:4000/health" ]
+      interval: 3s
+      timeout: 3s
+      retries: 5
+
+  grafana:
+    image: docker.io/grafana/grafana:11.1.0
+    container_name: grafana
+    ports:
+      - 3000:3000
+    environment:
+     - GF_INSTALL_PLUGINS=marcusolsson-treemap-panel, https://github.com/GreptimeTeam/greptimedb-grafana-datasource/releases/latest/download/info8fcc-greptimedb-datasource.zip;info8fcc
+    networks:
+      - demo-network
+    user: "$UID:$GID"
+    volumes:
+      - ./grafana_provisioning:/etc/grafana/provisioning
+    depends_on:
+      greptimedb:
+        condition: service_healthy
+
+  init_pipeline:
+    image: docker.io/alpine/curl
+    container_name: init_pipeline
+    networks:
+      - demo-network
+    depends_on:
+      greptimedb:
+        condition: service_healthy
+    volumes:
+      - ./config_data:/config_data
+    command: curl -X "POST" "http://greptimedb:4000/v1/events/pipelines/demo_pipeline" -F "file=@/config_data/greptime_pipeline.yaml" -v
+
+  vector:
+    build:
+      context: ./build_vector
+      dockerfile: ./Dockerfile
+    container_name: vector
+    networks:
+      - demo-network
+    volumes:
+      - ./config_data:/config_data
+      - shared-content:/logs
+    depends_on:
+      greptimedb:
+        condition: service_healthy
+      init_python_filesource:
+        condition: service_started
+    environment:
+      VECTOR_CONFIG_LOCATION: /config_data/vector.toml
+
+  nginx:
+    image: docker.io/nginx:1.27.0
+    container_name: nginx
+    ports:
+      - 8080:80
+      - 8081:8081
+    networks:
+      - demo-network
+    volumes:
+      - ./nginx_conf:/etc/nginx/templates
+      - shared-content:/var/log/nginx
+      - ./webcontent:/usr/share/nginx/html:ro
+    environment:
+      - NGINX_PORT=8080
+
+  prometheus:
+    image: docker.io/prom/prometheus:v2.45.6
+    container_name: prometheus_agent
+    ports:
+      - 9090:9090
+    networks:
+      - demo-network
+    volumes:
+      - ./config_data/prometheus-greptimedb.yml:/etc/prometheus/prometheus.yml:ro
+
+  nginx_exporter:
+    image: docker.io/nginx/nginx-prometheus-exporter:1.1
+    container_name: nginx_exporter
+    ports:
+      - 9113
+    command:
+      - '--nginx.scrape-uri=http://nginx:8081/nginx_status'
+    networks:
+      - demo-network
+    depends_on:
+      greptimedb:
+        condition: service_healthy
+
+networks:
+  demo-network:
+
+volumes:
+  shared-content:

nginx-log-metrics/grafana_provisioning/dashboards/dashboards.yml

@@ -0,0 +1,23 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'greptimedb demo dashboards'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> name of the dashboard folder.
+    folder: ''
+    # <string> folder UID. will be automatically generated if not specified
+    folderUid: ''
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: false
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/provisioning/dashboards
+      # <bool> use folder names from filesystem to create folders in Grafana

nginx-log-metrics/grafana_provisioning/datasources/greptime_pg_ds.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: greptimedb_pg
+    type: postgres
+    url: greptimedb:4003
+    jsonData:
+      database: public
+      sslmode: 'disable' # disable/require/verify-ca/verify-full

nginx-log-metrics/grafana_provisioning/datasources/greptime_plugin_ds.yml

@@ -0,0 +1,15 @@
+apiVersion: 1
+
+datasources:
+  - name: 'greptimedb'
+    type: 'info8fcc-greptimedb-datasource'
+    access: proxy
+    isDefault: false
+    orgId: 1
+    version: 1
+    editable: true
+    url: http://greptimedb:4000
+    jsonData:
+      path: '/resources'
+    secureJsonData:
+      apiKey: 'api-key'

nginx-log-metrics/nginx_conf/stub-status.conf.template

@@ -0,0 +1,8 @@
+server {
+  listen 8081;
+
+  location /nginx_status {
+    stub_status;
+    access_log off;
+  }
+}

nginx-log-metrics/webcontent/index.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <h1>It works</h1>
+  </body>
+</html>