From 808912db8794c0a89cdea960143ae7e73be93d11 Mon Sep 17 00:00:00 2001 From: Clement Date: Tue, 27 Aug 2024 21:18:16 +0200 Subject: [PATCH] feat(docker): add observability stack (#14203) --- tools/docker/README.md | 5 ++ tools/docker/alertmanager/alertmanager.yml | 15 +++++ tools/docker/compose | 13 +++- .../docker/docker-compose.observability.yaml | 44 ++++++++++++++ tools/docker/grafana/config.monitoring | 2 + .../provisioning/datasources/datasource.yml | 59 +++++++++++++++++++ tools/docker/prometheus/prometheus.yaml | 13 ++++ 7 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 tools/docker/alertmanager/alertmanager.yml create mode 100644 tools/docker/docker-compose.observability.yaml create mode 100644 tools/docker/grafana/config.monitoring create mode 100644 tools/docker/grafana/provisioning/datasources/datasource.yml create mode 100644 tools/docker/prometheus/prometheus.yaml diff --git a/tools/docker/README.md b/tools/docker/README.md index e0ccadd68c3..527951e64cc 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -23,6 +23,7 @@ cd tools/docker ### Compose script env vars The following env vars are used for the compose script : +- `WITH_OBSERVABILITY=true` to enable grafana, prometheus and alertmanager - `GETH_MODE=true` to use geth instead of parity - `CHAIN_ID=` to specify the chainID (default is 34055 for parity and 1337 for geth) - `HTTPURL=` to specify the RPC node HTTP url (default is set if you use geth or parity) @@ -35,6 +36,10 @@ for example : CHAIN_ID=11155111 WSURL=wss://eth.sepolia HTTPURL=https://eth.sepolia ./compose dev ``` +```sh +WITH_OBSERVABILITY=true ./compose up +``` + ## Dev Will run one node with a postgres database and by default a devnet RPC node that can be either geth or parity. diff --git a/tools/docker/alertmanager/alertmanager.yml b/tools/docker/alertmanager/alertmanager.yml new file mode 100644 index 00000000000..7521fd768b8 --- /dev/null +++ b/tools/docker/alertmanager/alertmanager.yml @@ -0,0 +1,15 @@ + +route: + receiver: 'mail' + repeat_interval: 4h + group_by: [ alertname ] + + +receivers: + - name: 'mail' + email_configs: + - smarthost: 'smtp.gmail.com:465' + auth_username: 'your_mail@gmail.com' + auth_password: "" + from: 'your_mail@gmail.com' + to: 'some_mail@gmail.com' \ No newline at end of file diff --git a/tools/docker/compose b/tools/docker/compose index 0e754a5ffcb..abaf3773336 100755 --- a/tools/docker/compose +++ b/tools/docker/compose @@ -16,7 +16,14 @@ if [ -z "$WSURL" ] && [ -z "$HTTPURL" ]; then fi fi -base="docker-compose $base_files" +args="node" + +if [ "$WITH_OBSERVABILITY" ]; then + base_files="$base_files -f docker-compose.observability.yaml" + args="$args grafana" +fi + +base="docker compose $base_files" dev="$base -f docker-compose.dev.yaml" configure() { @@ -41,7 +48,7 @@ configure() { clean_docker() { $base down -v --remove-orphans $dev down -v --remove-orphans - rm -f config.toml + rm -rf config.toml } usage() { @@ -71,7 +78,7 @@ logs) dev) configure $dev build - $dev up -d node + $dev up -d $args $dev watch --no-up node ;; connect) diff --git a/tools/docker/docker-compose.observability.yaml b/tools/docker/docker-compose.observability.yaml new file mode 100644 index 00000000000..c82c050d58e --- /dev/null +++ b/tools/docker/docker-compose.observability.yaml @@ -0,0 +1,44 @@ +services: + prometheus: + image: prom/prometheus:main + container_name: chainlink-prometheus + volumes: + - ./prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + restart: always + ports: + - 9090:9090 + grafana: + image: grafana/grafana:10.4.3 + user: "472" + depends_on: + - prometheus + - alertmanager + ports: + - 3000:3000 + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning/:/etc/grafana/provisioning/ + env_file: + - ./grafana/config.monitoring + restart: always + alertmanager: + image: prom/alertmanager:main + container_name: chainlink-alertmanager + volumes: + - "./alertmanager:/config" + - alertmanager-data:/data + command: --config.file=/config/alertmanager.yml --log.level=debug + restart: always + ports: + - 9093:9093 + +volumes: + alertmanager-data: {} + prometheus_data: {} + grafana_data: {} diff --git a/tools/docker/grafana/config.monitoring b/tools/docker/grafana/config.monitoring new file mode 100644 index 00000000000..a2b009166bd --- /dev/null +++ b/tools/docker/grafana/config.monitoring @@ -0,0 +1,2 @@ +GF_SECURITY_ADMIN_PASSWORD=foobar +GF_USERS_ALLOW_SIGN_UP=false \ No newline at end of file diff --git a/tools/docker/grafana/provisioning/datasources/datasource.yml b/tools/docker/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 00000000000..f57418b29f4 --- /dev/null +++ b/tools/docker/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,59 @@ +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Prometheus + orgId: 1 + +# list of datasources to insert/update depending +# whats available in the database +datasources: + # name of the datasource. Required + - name: Prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: false + # basic auth username, if used + basicAuthUser: + # basic auth password, if used + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: true + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: false + tlsAuthWithCACert: false + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: true + - name: Alertmanager + type: alertmanager + url: http://alertmanager:9093 + access: proxy + jsonData: + # Valid options for implementation include mimir, cortex and prometheus + implementation: prometheus + # Whether or not Grafana should send alert instances to this Alertmanager + handleGrafanaManagedAlerts: false \ No newline at end of file diff --git a/tools/docker/prometheus/prometheus.yaml b/tools/docker/prometheus/prometheus.yaml new file mode 100644 index 00000000000..96ca1064937 --- /dev/null +++ b/tools/docker/prometheus/prometheus.yaml @@ -0,0 +1,13 @@ +global: + scrape_interval: 5s +scrape_configs: + - job_name: 'local_scrape' + scrape_interval: 1s + static_configs: + - targets: ['chainlink-node:6688', 'chainlink-node-2:6688'] + metrics_path: '/metrics' +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: ['alertmanager:9093'] \ No newline at end of file