feat(docker): add observability stack (#14203)

tools/docker/README.md

@@ -23,6 +23,7 @@ cd tools/docker
 
 ### Compose script env vars
 The following env vars are used for the compose script :
+- `WITH_OBSERVABILITY=true` to enable grafana, prometheus and alertmanager
 - `GETH_MODE=true` to use geth instead of parity
 - `CHAIN_ID=<number>` to specify the chainID (default is 34055 for parity and 1337 for geth)
 - `HTTPURL=<url>` to specify the RPC node HTTP url (default is set if you use geth or parity)
@@ -35,6 +36,10 @@ for example :
 CHAIN_ID=11155111 WSURL=wss://eth.sepolia HTTPURL=https://eth.sepolia ./compose dev
 ```
 
+```sh
+WITH_OBSERVABILITY=true ./compose up
+```
+
 ## Dev
 
 Will run one node with a postgres database and by default a devnet RPC node that can be either geth or parity.

tools/docker/alertmanager/alertmanager.yml

@@ -0,0 +1,15 @@
+
+route:
+  receiver: 'mail'
+  repeat_interval: 4h
+  group_by: [ alertname ]
+
+
+receivers:
+  - name: 'mail'
+    email_configs:
+      - smarthost: 'smtp.gmail.com:465'
+        auth_username: '[email protected]'
+        auth_password: ""
+        from: '[email protected]'
+        to: '[email protected]'

tools/docker/compose

@@ -16,7 +16,14 @@ if [ -z "$WSURL" ] && [ -z "$HTTPURL" ]; then
   fi
 fi
 
-base="docker-compose $base_files"
+args="node"
+
+if [ "$WITH_OBSERVABILITY" ]; then
+    base_files="$base_files -f docker-compose.observability.yaml"
+    args="$args grafana"
+fi
+
+base="docker compose $base_files"
 dev="$base -f docker-compose.dev.yaml"
 
 configure() {
@@ -41,7 +48,7 @@ configure() {
 clean_docker() {
   $base down -v --remove-orphans
   $dev down -v --remove-orphans
-  rm -f config.toml
+  rm -rf config.toml
 }
 
 usage() {
@@ -71,7 +78,7 @@ logs)
 dev)
   configure
   $dev build
-  $dev up -d node
+  $dev up -d $args
   $dev watch --no-up node
   ;;
 connect)

tools/docker/docker-compose.observability.yaml

@@ -0,0 +1,44 @@
+services:
+  prometheus:
+    image: prom/prometheus:main
+    container_name: chainlink-prometheus
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yaml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    restart: always
+    ports:
+      - 9090:9090
+  grafana:
+    image: grafana/grafana:10.4.3
+    user: "472"
+    depends_on:
+      - prometheus
+      - alertmanager
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning/:/etc/grafana/provisioning/
+    env_file:
+      - ./grafana/config.monitoring
+    restart: always
+  alertmanager:
+    image: prom/alertmanager:main
+    container_name: chainlink-alertmanager
+    volumes:
+      - "./alertmanager:/config"
+      - alertmanager-data:/data
+    command: --config.file=/config/alertmanager.yml --log.level=debug
+    restart: always
+    ports:
+      - 9093:9093
+
+volumes:
+  alertmanager-data: {}
+  prometheus_data: {}
+  grafana_data: {}

tools/docker/grafana/config.monitoring

@@ -0,0 +1,2 @@
+GF_SECURITY_ADMIN_PASSWORD=foobar
+GF_USERS_ALLOW_SIGN_UP=false

tools/docker/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,59 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+  - name: Prometheus
+    # <string, required> datasource type. Required
+    type: prometheus
+    # <string, required> access mode. direct or proxy. Required
+    access: proxy
+    # <int> org id. will default to orgId 1 if not specified
+    orgId: 1
+    # <string> url
+    url: http://prometheus:9090
+    # <string> database password, if used
+    password:
+    # <string> database user, if used
+    user:
+    # <string> database name, if used
+    database:
+    # <bool> enable/disable basic auth
+    basicAuth: false
+    # <string> basic auth username, if used
+    basicAuthUser:
+    # <string> basic auth password, if used
+    basicAuthPassword:
+    # <bool> enable/disable with credentials headers
+    withCredentials:
+    # <bool> mark as default datasource. Max one per org
+    isDefault: true
+    # <map> fields that will be converted to json and stored in json_data
+    jsonData:
+      graphiteVersion: "1.1"
+      tlsAuth: false
+      tlsAuthWithCACert: false
+    # <string> json object of data that will be encrypted.
+    secureJsonData:
+      tlsCACert: "..."
+      tlsClientCert: "..."
+      tlsClientKey: "..."
+    version: 1
+    # <bool> allow users to edit datasources from the UI.
+    editable: true
+  - name: Alertmanager
+    type: alertmanager
+    url: http://alertmanager:9093
+    access: proxy
+    jsonData:
+      # Valid options for implementation include mimir, cortex and prometheus
+      implementation: prometheus
+      # Whether or not Grafana should send alert instances to this Alertmanager
+      handleGrafanaManagedAlerts: false

tools/docker/prometheus/prometheus.yaml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 5s
+scrape_configs:
+  - job_name: 'local_scrape'
+    scrape_interval: 1s
+    static_configs:
+      - targets: ['chainlink-node:6688', 'chainlink-node-2:6688']
+    metrics_path: '/metrics'
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets: ['alertmanager:9093']