diff --git a/.gitignore b/.gitignore index 3c549bd2..e404c37e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ local/ - +.env-old +plugins diff --git a/README.md b/README.md index d2b2df37..504160a9 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,15 @@ In demo/docker-compose.yml, set values for these keys. The key `INFLUXDB_BUCKET_ARCHIVE` is optional; if set, it should point to an InfluxDB bucket with longer retention policy than `INFLUXDB_BUCKET`, so that the "Archive Trace" button in Jaeger works properly: -```yaml -INFLUXDB_ADDR: -INFLUXDB_BUCKET: otel -INFLUXDB_BUCKET_ARCHIVE: otel-archive -INFLUXDB_TOKEN: + +The community addition focuses on the useability of the demo with Grafana. With this being said to improve demo setup we have introduced a `.env` file that will allow you to set the following variables: + +```bash +INFLUXDB_ADDR=eu-central-1-1.aws.cloud2.influxdata.com +INFLUXDB_TOKEN=xxxxxxx +INFLUXDB_ORG=xxxxxxxx +INFLUXDB_BUCKET=otel +INFLUXDB_BUCKET_ARCHIVE=otel-archive ``` In demo/otelcol-config.yml, set the similar values for these keys: @@ -29,7 +33,7 @@ token: Run the docker compose: ```console -$ docker compose --file demo/docker-compose.yml --project-directory . up --abort-on-container-exit --remove-orphans +$ docker-compose --file demo/docker-compose.yml --project-directory . up --abort-on-container-exit --remove-orphans ``` Traces are generated by "HotRod", an application designed to demonstrate tracing. @@ -43,6 +47,13 @@ Click any trace. View the dependency graph. Click "System Architecture". +Grafana is available at http://localhost:3000. The default username and password are both `admin`. The default datasource of flightSQL is already confgiured. + +**Note: You can find a dashboard to import under `demo/grafana/dashboards/Open Telemetry-1681814438598.json`** + +If you would like to access the Trace node tree. Then Make sure to enable it within the Jaeger datasource. Head to data sources and click on the Jaeger datasource. Then enable `Enable Node Graph`. Then click save and test. + + The images `otelcol-influxdb` and `jaeger-influxdb` are automatically built and pushed to Docker at https://hub.docker.com/r/jacobmarble/otelcol-influxdb and https://hub.docker.com/r/jacobmarble/jaeger-influxdb . ## Schema Reference diff --git a/demo/.env b/demo/.env new file mode 100644 index 00000000..f45a3623 --- /dev/null +++ b/demo/.env @@ -0,0 +1,5 @@ +INFLUXDB_ADDR=eu-central-1-1.aws.cloud2.influxdata.com +INFLUXDB_TOKEN=xxxxxxxx +INFLUXDB_ORG=xxxxxxx +INFLUXDB_BUCKET=otel +INFLUXDB_BUCKET_ARCHIVE= diff --git a/demo/docker-compose-development.yml b/demo/docker-compose-development.yml index 5ecb154e..1ae953e9 100644 --- a/demo/docker-compose-development.yml +++ b/demo/docker-compose-development.yml @@ -8,7 +8,6 @@ services: depends_on: - jaeger-influxdb environment: - #QUERY_BEARER_TOKEN_PROPAGATION: true LOG_LEVEL: warn SPAN_STORAGE_TYPE: grpc-plugin GRPC_STORAGE_SERVER: jaeger-influxdb:17271 @@ -27,17 +26,11 @@ services: image: jaeger-influxdb:local stop_grace_period: 10s environment: - LOG_LEVEL: info + LOG_LEVEL: debug LISTEN_ADDR: :17271 INFLUXDB_TIMEOUT: 30s - # required: hostname or hostname:port - INFLUXDB_ADDR: - # required: bucket name - INFLUXDB_BUCKET: otel - # optional: bucket name for archived traces - INFLUXDB_BUCKET_ARCHIVE: - # required - INFLUXDB_TOKEN: + env_file: + - demo/.env hotrod: build: @@ -47,18 +40,36 @@ services: stop_grace_period: 1s ports: - "8080:8080" # web UI + - "8083:8083" depends_on: - otelcol-influxdb environment: JAEGER_AGENT_HOST: otelcol-influxdb JAEGER_AGENT_PORT: 6831 + command: ["all", "-m", "prometheus"] otelcol-influxdb: build: context: . dockerfile: otelcol-influxdb/Dockerfile image: otelcol-influxdb:local - command: [ "--config", "/config.yml" ] + command: ["--config", "/config.yml"] stop_grace_period: 10s volumes: - ./demo/otelcol-config.yml:/config.yml:ro + env_file: + - demo/.env + + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" # web UI + environment: + - INFLUX_HOST=${INFLUXDB_ADDR} + - INFLUX_TOKEN=${INFLUXDB_TOKEN} + - INFLUX_ORG=${INFLUXDB_ORG} + - INFLUX_BUCKET=${INFLUXDB_BUCKET} + - GF_INSTALL_PLUGINS=influxdata-flightsql-datasource + volumes: + - ./demo/grafana/datasources:/etc/grafana/provisioning/datasources:ro + - ./demo/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index 31d0631c..2008f86c 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -8,7 +8,7 @@ services: depends_on: - jaeger-influxdb environment: - LOG_LEVEL: info + LOG_LEVEL: warn SPAN_STORAGE_TYPE: grpc-plugin GRPC_STORAGE_SERVER: jaeger-influxdb:17271 GRPC_STORAGE_CONNECTION_TIMEOUT: 30s @@ -19,35 +19,47 @@ services: - ./demo/jaeger-ui-config.json:/jaeger-ui-config.json:ro jaeger-influxdb: - image: jacobmarble/jaeger-influxdb:0.5.2 + image: jacobmarble/jaeger-influxdb:0.5.8 stop_grace_period: 10s environment: LOG_LEVEL: info LISTEN_ADDR: :17271 INFLUXDB_TIMEOUT: 30s - # required: hostname or hostname:port - INFLUXDB_ADDR: - # required: bucket name - INFLUXDB_BUCKET: otel - # optional: bucket name for archived traces - INFLUXDB_BUCKET_ARCHIVE: - # required - INFLUXDB_TOKEN: + env_file: + - demo/.env hotrod: image: jaegertracing/example-hotrod:1.41 stop_grace_period: 1s ports: - "8080:8080" # web UI + - "8083:8083" depends_on: - otelcol-influxdb environment: JAEGER_AGENT_HOST: otelcol-influxdb JAEGER_AGENT_PORT: 6831 + command: ["all", "-m", "prometheus"] otelcol-influxdb: image: otel/opentelemetry-collector-contrib:0.85.0 - command: [ "--config", "/config.yml" ] + command: ["--config", "/config.yml"] stop_grace_period: 10s volumes: - ./demo/otelcol-config.yml:/config.yml:ro + env_file: + - demo/.env + + grafana: + image: grafana/grafana:latest + ports: + - 3000:3000 + environment: + - INFLUX_HOST=${INFLUXDB_ADDR} + - INFLUX_TOKEN=${INFLUXDB_TOKEN} + - INFLUX_ORG=${INFLUXDB_ORG} + - INFLUX_BUCKET=${INFLUXDB_BUCKET} + - GF_INSTALL_PLUGINS=influxdata-flightsql-datasource + volumes: + - ./demo/grafana/datasources:/etc/grafana/provisioning/datasources:ro + - ./demo/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro diff --git a/demo/grafana/dashboards/Open Telemetry-1682672178148.json b/demo/grafana/dashboards/Open Telemetry-1682672178148.json new file mode 100644 index 00000000..2f345b14 --- /dev/null +++ b/demo/grafana/dashboards/Open Telemetry-1682672178148.json @@ -0,0 +1,641 @@ +{ + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.5.1" + }, + { + "type": "panel", + "id": "histogram", + "name": "Histogram", + "version": "" + }, + { + "type": "datasource", + "id": "influxdata-flightsql-datasource", + "name": "FlightSQL", + "version": "1.0.1" + }, + { + "type": "datasource", + "id": "jaeger", + "name": "Jaeger", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "nodeGraph", + "name": "Node Graph", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "traces", + "name": "Traces", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [ + { + "title": "", + "url": "/d/${__dashboard.uid}?var-Service=${__value.raw}&from=${__url_time_from}&to=${__url_time_to}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "service.name" + } + ] + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "format": "table", + "queryText": "SELECT DISTINCT(\"service.name\") FROM \"spans\" WHERE $__timeRange(time)", + "rawEditor": true, + "rawQuery": true, + "refId": "A" + } + ], + "title": "Services", + "type": "table" + }, + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "blue", + "value": 30 + }, + { + "color": "dark-orange", + "value": 50 + }, + { + "color": "dark-red", + "value": 70 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 5, + "y": 0 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.3, + "colorByField": "duration (lastNotNull)", + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 5 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "format": "table", + "queryText": "SELECT \"duration_nano\"/ 1000000 as \"duration\", \"span_id\" ,time FROM \"spans\" WHERE $__timeRange(time) AND \"service.name\" = '$Service' order by time", + "rawEditor": true, + "rawQuery": true, + "refId": "A" + } + ], + "title": "Duration", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "duration": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "duration_nano": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "span_id": { + "aggregations": [], + "operation": "groupby" + }, + "time": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 75 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 14, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^error_percentage$/", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "influxdata-flightsql-datasource", + "uid": "FlightSQL" + }, + "format": "table", + "queryText": "SELECT\n (COUNT(CASE WHEN \"otel.status_code\" = 'Error' THEN 1 END) * 100.0) / COUNT(*) as error_percentage FROM spans\n WHERE $__timeRange(time) AND \"service.name\" = '$Service' ", + "rawEditor": true, + "rawQuery": true, + "refId": "A" + } + ], + "title": "Error Rate", + "transformations": [], + "type": "gauge" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 19, + "x": 5, + "y": 8 + }, + "id": 13, + "options": { + "bucketOffset": 0, + "combine": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "queryType": "search", + "refId": "A", + "service": "$Service" + } + ], + "title": "Service Latency Histogram", + "transformations": [], + "type": "histogram" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Show Details", + "url": "/d/${__dashboard.uid}?var-TraceID=${__value.raw}&var-Service=${Service}&from=${__url_time_from}&to=${__url_time_to}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 13 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "queryType": "search", + "refId": "A", + "service": "$Service" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 10, + "y": 13 + }, + "id": 11, + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "query": "$TraceID", + "refId": "A" + } + ], + "title": "Relationships ", + "type": "nodeGraph" + }, + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 9, + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "Jaeger" + }, + "query": "$TraceID", + "refId": "A" + } + ], + "title": "Trace", + "type": "traces" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "redis", + "value": "redis" + }, + "hide": 0, + "label": "Service", + "name": "Service", + "options": [ + { + "selected": true, + "text": "redis", + "value": "redis" + } + ], + "query": "redis", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "120011cb7f5fefbd", + "value": "120011cb7f5fefbd" + }, + "hide": 0, + "label": "TraceID", + "name": "TraceID", + "options": [ + { + "selected": true, + "text": "120011cb7f5fefbd", + "value": "120011cb7f5fefbd" + } + ], + "query": "120011cb7f5fefbd", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Open Telemetry", + "uid": "jJwDAiE4z", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/demo/grafana/dashboards/dashboards.yml b/demo/grafana/dashboards/dashboards.yml new file mode 100644 index 00000000..adaf1d93 --- /dev/null +++ b/demo/grafana/dashboards/dashboards.yml @@ -0,0 +1,24 @@ +apiVersion: 1 + +providers: + # an unique provider name. Required + - name: 'OpenTelemetry Demo' + # Org id. Default to 1 + orgId: 1 + # name of the dashboard folder. + folder: 'general' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /etc/grafana/provisioning/dashboards + # use folder names from filesystem to create folders in Grafana + foldersFromFilesStructure: true \ No newline at end of file diff --git a/demo/grafana/datasources/flightsql.yml b/demo/grafana/datasources/flightsql.yml new file mode 100644 index 00000000..3ccf9e35 --- /dev/null +++ b/demo/grafana/datasources/flightsql.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: FlightSQL + type: influxdata-flightsql-datasource + typeName: FlightSQL + access: proxy + url: '' + user: '' + database: '' + basicAuth: false + isDefault: true + jsonData: + host: ${INFLUX_HOST}:443 + metadata: + - bucket-name: ${INFLUX_BUCKET} + secure: true + token: ${INFLUX_TOKEN} + readOnly: false + editable: true + diff --git a/demo/grafana/datasources/jaeger.yml b/demo/grafana/datasources/jaeger.yml new file mode 100644 index 00000000..1f41fccc --- /dev/null +++ b/demo/grafana/datasources/jaeger.yml @@ -0,0 +1,19 @@ +apiVersion: 1 +datasources: + - name: Jaeger + type: jaeger + access: proxy + url: http://jaeger-query:16686 + readOnly: false + editable: true + isDefault: false + jsonData: + tracesToLogs: + # Field with internal link pointing to a logs data source in Grafana. + # datasourceUid value must match the datasourceUid value of the logs data source. + datasourceUid: 'grafana' + filterByTraceID: true + filterBySpanID: false + nodeGraph: + enabled: true + diff --git a/demo/otelcol-config.yml b/demo/otelcol-config.yml index e3fabb11..ce91f50c 100644 --- a/demo/otelcol-config.yml +++ b/demo/otelcol-config.yml @@ -11,9 +11,9 @@ receivers: exporters: influxdb: - endpoint: - bucket: otel - token: + endpoint: https://${INFLUXDB_ADDR}/ + bucket: ${INFLUXDB_BUCKET} + token: ${INFLUXDB_TOKEN} metrics_schema: otel-v1 connectors: