From 412b5269eafac5d9e214ad387afd64e3dac9450e Mon Sep 17 00:00:00 2001 From: Kevin K Biju <52661649+heavycrystal@users.noreply.github.com> Date: Wed, 27 Sep 2023 20:54:49 +0530 Subject: [PATCH] added overall throughput metric and fixed records pulled (#437) --- docker-compose.yml | 2 +- flow/activities/flowable.go | 12 +- flow/connectors/postgres/postgres.go | 13 +- flow/connectors/utils/metrics/metrics.go | 20 +- stacks/grafana/flow_monitoring_dashboard.json | 1007 ++++++++--------- .../grafana/flow_monitoring_dashboard_v0.json | 585 ++++++++++ 6 files changed, 1100 insertions(+), 539 deletions(-) create mode 100644 stacks/grafana/flow_monitoring_dashboard_v0.json diff --git a/docker-compose.yml b/docker-compose.yml index a5c527b056..75e1b8234d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -222,7 +222,7 @@ services: context: . dockerfile: stacks/ui.Dockerfile ports: - - 3000:3000 + - 3001:3000 environment: <<: *catalog-config PEERDB_FLOW_SERVER_ADDRESS: flow_api:8112 diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index 5011845a23..25b73e2ef9 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -10,6 +10,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" "github.com/PeerDB-io/peer-flow/connectors/utils" + "github.com/PeerDB-io/peer-flow/connectors/utils/metrics" "github.com/PeerDB-io/peer-flow/connectors/utils/monitoring" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" @@ -238,6 +239,9 @@ func (a *FlowableActivity) StartFlow(ctx context.Context, log.WithFields(log.Fields{ "flowName": input.FlowConnectionConfigs.FlowJobName, }).Info("no records to push") + metrics.LogSyncMetrics(ctx, input.FlowConnectionConfigs.FlowJobName, 0, 1) + metrics.LogNormalizeMetrics(ctx, input.FlowConnectionConfigs.FlowJobName, 0, 1, 0) + metrics.LogCDCRawThroughputMetrics(ctx, input.FlowConnectionConfigs.FlowJobName, 0) return &model.SyncResponse{ RelationMessageMapping: recordsWithTableSchemaDelta.RelationMessageMapping, TableSchemaDelta: recordsWithTableSchemaDelta.TableSchemaDelta, @@ -294,6 +298,9 @@ func (a *FlowableActivity) StartFlow(ctx context.Context, pushedRecordsWithCount := fmt.Sprintf("pushed %d records", numRecords) activity.RecordHeartbeat(ctx, pushedRecordsWithCount) + metrics.LogCDCRawThroughputMetrics(ctx, input.FlowConnectionConfigs.FlowJobName, + float64(numRecords)/(pullDuration.Seconds()+syncDuration.Seconds())) + return res, nil } @@ -317,8 +324,11 @@ func (a *FlowableActivity) StartNormalize( return nil, fmt.Errorf("failed to get last sync batch ID: %v", err) } - return nil, a.CatalogMirrorMonitor.UpdateEndTimeForCDCBatch(ctx, input.FlowConnectionConfigs.FlowJobName, + err = a.CatalogMirrorMonitor.UpdateEndTimeForCDCBatch(ctx, input.FlowConnectionConfigs.FlowJobName, lastSyncBatchID) + if err != nil { + return nil, err + } } else if err != nil { return nil, err } diff --git a/flow/connectors/postgres/postgres.go b/flow/connectors/postgres/postgres.go index 20ebe09588..46f946e77b 100644 --- a/flow/connectors/postgres/postgres.go +++ b/flow/connectors/postgres/postgres.go @@ -223,16 +223,19 @@ func (c *PostgresConnector) PullRecords(req *model.PullRecordsRequest) (*model.R return nil, fmt.Errorf("failed to create cdc source: %w", err) } + startTime := time.Now() recordsWithSchemaDelta, err := cdc.PullRecords(req) if err != nil { return nil, err } + + totalRecordsAtSource, err := c.getApproxTableCounts(maps.Keys(req.TableNameMapping)) + if err != nil { + return nil, err + } + metrics.LogPullMetrics(c.ctx, req.FlowJobName, recordsWithSchemaDelta.RecordBatch, + totalRecordsAtSource, time.Since(startTime)) if len(recordsWithSchemaDelta.RecordBatch.Records) > 0 { - totalRecordsAtSource, err := c.getApproxTableCounts(maps.Keys(req.TableNameMapping)) - if err != nil { - return nil, err - } - metrics.LogPullMetrics(c.ctx, req.FlowJobName, recordsWithSchemaDelta.RecordBatch, totalRecordsAtSource) cdcMirrorMonitor, ok := c.ctx.Value(shared.CDCMirrorMonitorKey).(*monitoring.CatalogMirrorMonitor) if ok { latestLSN, err := c.getCurrentLSN() diff --git a/flow/connectors/utils/metrics/metrics.go b/flow/connectors/utils/metrics/metrics.go index 19a364dd1f..02f30fd366 100644 --- a/flow/connectors/utils/metrics/metrics.go +++ b/flow/connectors/utils/metrics/metrics.go @@ -15,6 +15,7 @@ func LogPullMetrics( flowJobName string, recordBatch *model.RecordBatch, totalRecordsAtSource int64, + duration time.Duration, ) { if ctx.Value(shared.EnableMetricsKey) != true { return @@ -39,10 +40,10 @@ func LogPullMetrics( } } - insertRecordsPulledGauge.Update(float64(insertRecords)) - updateRecordsPulledGauge.Update(float64(updateRecords)) - deleteRecordsPulledGauge.Update(float64(deleteRecords)) - totalRecordsPulledGauge.Update(float64(len(recordBatch.Records))) + insertRecordsPulledGauge.Update(float64(insertRecords) / duration.Seconds()) + updateRecordsPulledGauge.Update(float64(updateRecords) / duration.Seconds()) + deleteRecordsPulledGauge.Update(float64(deleteRecords) / duration.Seconds()) + totalRecordsPulledGauge.Update(float64(len(recordBatch.Records)) / duration.Seconds()) totalRecordsAtSourceGauge.Update(float64(totalRecordsAtSource)) } @@ -118,3 +119,14 @@ func LogQRepNormalizeMetrics(ctx context.Context, flowJobName string, recordsSyncedPerSecondGauge.Update(float64(normalizedRecordsCount) / duration.Seconds()) totalRecordsAtTargetGauge.Update(float64(totalRecordsAtTarget)) } + +func LogCDCRawThroughputMetrics(ctx context.Context, flowJobName string, throughput float64) { + if ctx.Value(shared.EnableMetricsKey) != true { + return + } + + metricsHandler := activity.GetMetricsHandler(ctx) + totalThroughputGauge := + metricsHandler.Gauge(fmt.Sprintf("cdcflow.%s.records_throughput", flowJobName)) + totalThroughputGauge.Update(throughput) +} diff --git a/stacks/grafana/flow_monitoring_dashboard.json b/stacks/grafana/flow_monitoring_dashboard.json index 722f757fcf..0c1477ff82 100644 --- a/stacks/grafana/flow_monitoring_dashboard.json +++ b/stacks/grafana/flow_monitoring_dashboard.json @@ -1,585 +1,536 @@ { - "__inputs": [ - { - "name": "peerdb_prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.2" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ { - "builtIn": 1, "datasource": { - "type": "grafana", - "uid": "-- Grafana --" + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "thresholdsStyle": { - "mode": "off" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "overrides": [] }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 11, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 0 }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_records_synced_per_second", - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" - } - ], - "title": "records synced / second", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_synced_per_second", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "records synced / second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P8C2E8E0157474F52" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "thresholdsStyle": { - "mode": "off" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "flow_worker_cdcflow_schemaa_records_throughput" + ], + "prefix": "All except:", + "readOnly": true + } }, - { - "color": "red", - "value": 80 - } - ] + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 13, + "x": 11, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "overrides": [ + "targets": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "difference_in_record_counts" - ], - "prefix": "All except:", - "readOnly": true - } + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_throughput", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" } - ] - }, - "gridPos": { - "h": 10, - "w": 13, - "x": 11, - "y": 0 + ], + "title": "overall mirror throughput", + "transformations": [], + "type": "timeseries" }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_records_at_source", - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 10 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_records_at_target", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "difference in records between source and target", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "difference_in_record_counts", - "binary": { - "left": "flow_worker_${job_type}_${job_name}_records_at_source", - "operator": "-", - "reducer": "sum", - "right": "flow_worker_${job_type}_${job_name}_records_at_target" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_normalized_per_second", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" + ], + "title": "records normalized / second", + "type": "timeseries" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "thresholdsStyle": { - "mode": "off" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "overrides": [] }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 11, - "x": 0, - "y": 10 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "gridPos": { + "h": 10, + "w": 13, + "x": 11, + "y": 10 }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_records_normalized_per_second", - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" - } - ], - "title": "records normalized / second", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_insert_records_pulled", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_update_records_pulled", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" }, - "thresholdsStyle": { - "mode": "off" - } + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_delete_records_pulled", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_total_records_pulled", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "D" } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 13, - "x": 11, - "y": 10 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ + ], + "title": "records pulled / second with types", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" + "current": { + "selected": false, + "text": "schemaa", + "value": "schemaa" }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_insert_records_pulled", - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" - }, - { "datasource": { "type": "prometheus", "uid": "peerdb_prometheus" }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_update_records_pulled", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" + "definition": "metrics(flow_worker_${job_type}_.*_total_records_pulled)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job_name", + "options": [], + "query": { + "query": "metrics(flow_worker_${job_type}_.*_total_records_pulled)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_delete_records_pulled", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "C" + "refresh": 1, + "regex": "flow_worker_${job_type}_(?.*)_total_records_pulled", + "skipUrlSync": false, + "sort": 0, + "type": "query" }, { + "current": { + "selected": false, + "text": "cdcflow", + "value": "cdcflow" + }, "datasource": { "type": "prometheus", "uid": "peerdb_prometheus" }, - "editorMode": "code", - "expr": "flow_worker_${job_type}_${job_name}_total_records_pulled", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "D" + "definition": "metrics(flow_worker_.*_total_records_pulled)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job_type", + "options": [], + "query": { + "query": "metrics(flow_worker_.*_total_records_pulled)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "flow_worker_(?.*flow)_.*", + "skipUrlSync": false, + "sort": 0, + "type": "query" } - ], - "title": "records pulled / second with types", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "definition": "metrics(flow_worker_${job_type}_.*_total_records_pulled)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "job_name", - "options": [], - "query": { - "query": "metrics(flow_worker_${job_type}_.*_total_records_pulled)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "flow_worker_${job_type}_(?.*)_total_records_pulled", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "peerdb_prometheus" - }, - "definition": "metrics(flow_worker_.*_total_records_pulled)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "job_type", - "options": [], - "query": { - "query": "metrics(flow_worker_.*_total_records_pulled)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "flow_worker_(?.*flow)_.*", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "PeerDB mirror monitoring dashboard", - "uid": "cac849d7-5353-4bd2-8f4f-925ad428cf1d", - "version": 11, - "weekStart": "" -} \ No newline at end of file + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PeerDB mirror monitoring dashboard", + "uid": "cac849d7-5353-4bd2-8f4f-925ad428cf1d", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/stacks/grafana/flow_monitoring_dashboard_v0.json b/stacks/grafana/flow_monitoring_dashboard_v0.json new file mode 100644 index 0000000000..722f757fcf --- /dev/null +++ b/stacks/grafana/flow_monitoring_dashboard_v0.json @@ -0,0 +1,585 @@ +{ + "__inputs": [ + { + "name": "peerdb_prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_synced_per_second", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "records synced / second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "difference_in_record_counts" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 13, + "x": 11, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_at_source", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_at_target", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "difference in records between source and target", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "difference_in_record_counts", + "binary": { + "left": "flow_worker_${job_type}_${job_name}_records_at_source", + "operator": "-", + "reducer": "sum", + "right": "flow_worker_${job_type}_${job_name}_records_at_target" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 10 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_records_normalized_per_second", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "records normalized / second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 13, + "x": 11, + "y": 10 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_insert_records_pulled", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_update_records_pulled", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_delete_records_pulled", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "editorMode": "code", + "expr": "flow_worker_${job_type}_${job_name}_total_records_pulled", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "D" + } + ], + "title": "records pulled / second with types", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "definition": "metrics(flow_worker_${job_type}_.*_total_records_pulled)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job_name", + "options": [], + "query": { + "query": "metrics(flow_worker_${job_type}_.*_total_records_pulled)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "flow_worker_${job_type}_(?.*)_total_records_pulled", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "peerdb_prometheus" + }, + "definition": "metrics(flow_worker_.*_total_records_pulled)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job_type", + "options": [], + "query": { + "query": "metrics(flow_worker_.*_total_records_pulled)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "flow_worker_(?.*flow)_.*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PeerDB mirror monitoring dashboard", + "uid": "cac849d7-5353-4bd2-8f4f-925ad428cf1d", + "version": 11, + "weekStart": "" +} \ No newline at end of file