diff --git a/.env b/.env new file mode 100644 index 00000000..a1119bbc --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +INFLUXDB_ADDR=eu-central-1-1.aws.cloud2.influxdata.com +INFLUXDB_TOKEN=xxxxxxxx +INFLUXDB_ORG=xxxxxxx +INFLUXDB_BUCKET=otel diff --git a/.gitignore b/.gitignore index 3c549bd2..e404c37e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ local/ - +.env-old +plugins diff --git a/README.md b/README.md index d2b2df37..504160a9 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,15 @@ In demo/docker-compose.yml, set values for these keys. The key `INFLUXDB_BUCKET_ARCHIVE` is optional; if set, it should point to an InfluxDB bucket with longer retention policy than `INFLUXDB_BUCKET`, so that the "Archive Trace" button in Jaeger works properly: -```yaml -INFLUXDB_ADDR: -INFLUXDB_BUCKET: otel -INFLUXDB_BUCKET_ARCHIVE: otel-archive -INFLUXDB_TOKEN: + +The community addition focuses on the useability of the demo with Grafana. With this being said to improve demo setup we have introduced a `.env` file that will allow you to set the following variables: + +```bash +INFLUXDB_ADDR=eu-central-1-1.aws.cloud2.influxdata.com +INFLUXDB_TOKEN=xxxxxxx +INFLUXDB_ORG=xxxxxxxx +INFLUXDB_BUCKET=otel +INFLUXDB_BUCKET_ARCHIVE=otel-archive ``` In demo/otelcol-config.yml, set the similar values for these keys: @@ -29,7 +33,7 @@ token: Run the docker compose: ```console -$ docker compose --file demo/docker-compose.yml --project-directory . up --abort-on-container-exit --remove-orphans +$ docker-compose --file demo/docker-compose.yml --project-directory . up --abort-on-container-exit --remove-orphans ``` Traces are generated by "HotRod", an application designed to demonstrate tracing. @@ -43,6 +47,13 @@ Click any trace. View the dependency graph. Click "System Architecture". +Grafana is available at http://localhost:3000. The default username and password are both `admin`. The default datasource of flightSQL is already confgiured. + +**Note: You can find a dashboard to import under `demo/grafana/dashboards/Open Telemetry-1681814438598.json`** + +If you would like to access the Trace node tree. Then Make sure to enable it within the Jaeger datasource. Head to data sources and click on the Jaeger datasource. Then enable `Enable Node Graph`. Then click save and test. + + The images `otelcol-influxdb` and `jaeger-influxdb` are automatically built and pushed to Docker at https://hub.docker.com/r/jacobmarble/otelcol-influxdb and https://hub.docker.com/r/jacobmarble/jaeger-influxdb . ## Schema Reference diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index 9cda0f44..041480e5 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -1,5 +1,5 @@ +version: '3' services: - jaeger-query: image: jaegertracing/jaeger-query:1.49 stop_grace_period: 10s @@ -22,28 +22,23 @@ services: image: jacobmarble/jaeger-influxdb:0.5.9 stop_grace_period: 10s environment: - LOG_LEVEL: info + LOG_LEVEL: debug LISTEN_ADDR: :17271 INFLUXDB_TIMEOUT: 30s - # required: hostname or hostname:port - INFLUXDB_ADDR: - # required: bucket name - INFLUXDB_BUCKET: otel - # optional: bucket name for archived traces - INFLUXDB_BUCKET_ARCHIVE: - # required - INFLUXDB_TOKEN: - + env_file: + - .env hotrod: image: jaegertracing/example-hotrod:1.41 stop_grace_period: 1s ports: - "8080:8080" # web UI + - "8083:8083" depends_on: - otelcol-influxdb environment: JAEGER_AGENT_HOST: otelcol-influxdb JAEGER_AGENT_PORT: 6831 + command: ["all", "-m", "prometheus"] otelcol-influxdb: image: otel/opentelemetry-collector-contrib:0.87.0 @@ -51,3 +46,23 @@ services: stop_grace_period: 10s volumes: - ./demo/otelcol-config.yml:/config.yml:ro + env_file: + - .env + + grafana: + image: grafana/grafana:latest + ports: + - 3000:3000 + environment: + - INFLUX_HOST=${INFLUXDB_ADDR} + - INFLUX_TOKEN=${INFLUXDB_TOKEN} + - INFLUX_ORG=${INFLUXDB_ORG} + - INFLUX_BUCKET=${INFLUXDB_BUCKET} + - GF_INSTALL_PLUGINS=influxdata-flightsql-datasource + volumes: + - ./demo/grafana/datasources:/etc/grafana/provisioning/datasources + - ./demo/grafana/dashboards:/etc/grafana/provisioning/dashboards + - grafana:/var/lib/grafana/ + restart: always +volumes: + grafana: \ No newline at end of file diff --git a/demo/grafana/dashboards/Open Telemetry-1682672178148.json b/demo/grafana/dashboards/Open Telemetry-1682672178148.json new file mode 100644 index 00000000..2f345b14 --- /dev/null +++ b/demo/grafana/dashboards/Open Telemetry-1682672178148.json @@ -0,0 +1,641 @@ +{ + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.5.1" + }, + { + "type": "panel", + "id": "histogram", + "name": "Histogram", + "version": "" + }, + { + "type": "datasource", + "id": "influxdata-flightsql-datasource", + "name": "FlightSQL", + "version": "1.0.1" + }, + { + "type": "datasource", + "id": "jaeger", + "name": "Jaeger", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "nodeGraph", + "name": "Node Graph", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "traces", + "name": "Traces", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [ + { + "title": "", + "url": "/d/${__dashboard.uid}?var-Service=${__value.raw}&from=${__url_time_from}&to=${__url_time_to}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "service.name" + } + ] + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "format": "table", + "queryText": "SELECT DISTINCT(\"service.name\") FROM \"spans\" WHERE $__timeRange(time)", + "rawEditor": true, + "rawQuery": true, + "refId": "A" + } + ], + "title": "Services", + "type": "table" + }, + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "blue", + "value": 30 + }, + { + "color": "dark-orange", + "value": 50 + }, + { + "color": "dark-red", + "value": 70 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 5, + "y": 0 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.3, + "colorByField": "duration (lastNotNull)", + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 5 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "format": "table", + "queryText": "SELECT \"duration_nano\"/ 1000000 as \"duration\", \"span_id\" ,time FROM \"spans\" WHERE $__timeRange(time) AND \"service.name\" = '$Service' order by time", + "rawEditor": true, + "rawQuery": true, + "refId": "A" + } + ], + "title": "Duration", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "duration": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "duration_nano": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "span_id": { + "aggregations": [], + "operation": "groupby" + }, + "time": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 75 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 14, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^error_percentage$/", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "format": "table", + "queryText": "SELECT\n (COUNT(CASE WHEN \"otel.status_code\" = 'Error' THEN 1 END) * 100.0) / COUNT(*) as error_percentage FROM spans\n WHERE $__timeRange(time) AND \"service.name\" = '$Service' ", + "rawEditor": true, + "rawQuery": true, + "refId": "A" + } + ], + "title": "Error Rate", + "transformations": [], + "type": "gauge" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 19, + "x": 5, + "y": 8 + }, + "id": 13, + "options": { + "bucketOffset": 0, + "combine": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "queryType": "search", + "refId": "A", + "service": "$Service" + } + ], + "title": "Service Latency Histogram", + "transformations": [], + "type": "histogram" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Show Details", + "url": "/d/${__dashboard.uid}?var-TraceID=${__value.raw}&var-Service=${Service}&from=${__url_time_from}&to=${__url_time_to}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 13 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "queryType": "search", + "refId": "A", + "service": "$Service" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 10, + "y": 13 + }, + "id": 11, + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "query": "$TraceID", + "refId": "A" + } + ], + "title": "Relationships ", + "type": "nodeGraph" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 9, + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "query": "$TraceID", + "refId": "A" + } + ], + "title": "Trace", + "type": "traces" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "redis", + "value": "redis" + }, + "hide": 0, + "label": "Service", + "name": "Service", + "options": [ + { + "selected": true, + "text": "redis", + "value": "redis" + } + ], + "query": "redis", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "120011cb7f5fefbd", + "value": "120011cb7f5fefbd" + }, + "hide": 0, + "label": "TraceID", + "name": "TraceID", + "options": [ + { + "selected": true, + "text": "120011cb7f5fefbd", + "value": "120011cb7f5fefbd" + } + ], + "query": "120011cb7f5fefbd", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Open Telemetry", + "uid": "jJwDAiE4z", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/demo/grafana/dashboards/dashboards.yml b/demo/grafana/dashboards/dashboards.yml new file mode 100644 index 00000000..adaf1d93 --- /dev/null +++ b/demo/grafana/dashboards/dashboards.yml @@ -0,0 +1,24 @@ +apiVersion: 1 + +providers: + # an unique provider name. Required + - name: 'OpenTelemetry Demo' + # Org id. Default to 1 + orgId: 1 + # name of the dashboard folder. + folder: 'general' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /etc/grafana/provisioning/dashboards + # use folder names from filesystem to create folders in Grafana + foldersFromFilesStructure: true \ No newline at end of file diff --git a/demo/grafana/datasources/flightsql.yml b/demo/grafana/datasources/flightsql.yml new file mode 100644 index 00000000..3ccf9e35 --- /dev/null +++ b/demo/grafana/datasources/flightsql.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: FlightSQL + type: influxdata-flightsql-datasource + typeName: FlightSQL + access: proxy + url: '' + user: '' + database: '' + basicAuth: false + isDefault: true + jsonData: + host: ${INFLUX_HOST}:443 + metadata: + - bucket-name: ${INFLUX_BUCKET} + secure: true + token: ${INFLUX_TOKEN} + readOnly: false + editable: true + diff --git a/demo/grafana/datasources/jaeger.yml b/demo/grafana/datasources/jaeger.yml new file mode 100644 index 00000000..1f41fccc --- /dev/null +++ b/demo/grafana/datasources/jaeger.yml @@ -0,0 +1,19 @@ +apiVersion: 1 +datasources: + - name: Jaeger + type: jaeger + access: proxy + url: http://jaeger-query:16686 + readOnly: false + editable: true + isDefault: false + jsonData: + tracesToLogs: + # Field with internal link pointing to a logs data source in Grafana. + # datasourceUid value must match the datasourceUid value of the logs data source. + datasourceUid: 'grafana' + filterByTraceID: true + filterBySpanID: false + nodeGraph: + enabled: true + diff --git a/demo/otelcol-config.yml b/demo/otelcol-config.yml index e3fabb11..ce91f50c 100644 --- a/demo/otelcol-config.yml +++ b/demo/otelcol-config.yml @@ -11,9 +11,9 @@ receivers: exporters: influxdb: - endpoint: - bucket: otel - token: + endpoint: https://${INFLUXDB_ADDR}/ + bucket: ${INFLUXDB_BUCKET} + token: ${INFLUXDB_TOKEN} metrics_schema: otel-v1 connectors: diff --git a/demo/telegraf/telegraf.conf b/demo/telegraf/telegraf.conf new file mode 100644 index 00000000..e9e70abe --- /dev/null +++ b/demo/telegraf/telegraf.conf @@ -0,0 +1,57 @@ +[global_tags] + +[agent] + interval = "30s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + omit_hostname = false + +[[outputs.influxdb_v2]] + urls = ["http://${INFLUXDB_ADDR}:8086"] # Replace with your InfluxDB v2 instance URL + token = "$[INFLUXDB_TOKEN}" # Replace with your InfluxDB v2 token + organization = "${INFLUXDB_ORG}" # Replace with your InfluxDB v2 organization name + bucket = "${INFLUXDB_BUCKET}" # Replace with your InfluxDB v2 bucket name + +[[inputs.prometheus]] + name_override = "metrics" + ## By specifying a "v2" string to the version, you can use the new v2 + ## metrics format. + metric_version = 2 + + ## An array of urls to scrape metrics from. + urls = ["http://localhost:8083/metrics"] # Replace with your Prometheus target URL + + ## An array of Kubernetes services to scrape metrics from. + # kubernetes_services = ["http://my-service-dns.my-namespace:9100/metrics"] + + ## Kubernetes config file to create client. + # kube_config = "/path/to/kubernetes.config" + + ## Scrape interval + # interval = "1m" + + ## Scrape metrics for kube-system services. + ## If set to true, it is recommended to limit your urls to avoid too much + ## data being scraped. + # monitor_kubernetes_pods = false + + [[processors.regex]] + namepass = ["metrics"] + + [[processors.regex.tags]] + key = "_measurement" + pattern = "(.*)_([^_]*)_([^_]*)_([^_]*)_([^_]*)" + replacement = "${1}_${2}_${3}_${4}" + result_key = "prefix" + [[processors.regex.tags]] + key = "_measurement" + pattern = "(.*)_([^_]*)_([^_]*)_([^_]*)_([^_]*)" + replacement = "${5}" + result_key = "suffix" \ No newline at end of file