diff --git a/docs/source/install/monitor-without-docker.rst b/docs/source/install/monitor-without-docker.rst index e4b3b1b78..b2b0d5c31 100644 --- a/docs/source/install/monitor-without-docker.rst +++ b/docs/source/install/monitor-without-docker.rst @@ -197,8 +197,8 @@ For example the scrape config for Scylla: .. code-block:: shell global: - scrape_interval: 5s # By default, scrape targets every 5 second. - scrape_timeout: 4s # Timeout before trying to scape a target again + scrape_interval: 20s # By default, scrape targets every 20s second. + scrape_timeout: 15s # Timeout before trying to scape a target again # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). @@ -308,6 +308,9 @@ For example the scrape config for Scylla: target_label: by replacement: 'instance,shard' +.. note:: + You can change scrape_interval to a longer interval to save resources. If you do that, you must change the Grafan configuration to match it. + 5. Create and set ``scylla_servers.yml`` file point to your Scylla nodes and ``scylla_manager_server.yml`` file to point to your Scylla Manager. .. note:: @@ -490,6 +493,8 @@ For example url: http://192.168.135.167:9090 access: proxy basicAuth: false + jsonData: + timeInterval: '20s' # Must match prometheus gloabal scrape_interval - name: alertmanager type: camptocamp-prometheus-alertmanager-datasource @@ -503,10 +508,10 @@ For example basicAuth: isDefault: jsonData: - severity_critical: '4' - severity_high: '3' - severity_warning: '2' - severity_info: '1' + implementation: 'prometheus' + +.. note:: + The timeInterval parameter must match Prometheus scrape_interval config. 6. Start the Grafana service diff --git a/docs/source/install/start-all.rst b/docs/source/install/start-all.rst index f270d092f..f9fd7f395 100644 --- a/docs/source/install/start-all.rst +++ b/docs/source/install/start-all.rst @@ -25,12 +25,14 @@ You cannot use port mapping when using the ``-l`` flag **--compose** This is an experimental option. When set, it initiates a two-stage process. Firstly, a ``docker-compose.yml`` file is created along with an ``.env`` file. Following this, the script executes ``docker-compose up``. Please note that having docker-compose is a prerequisite. +**--scrap scrap-interval** Allows changing Prometheus scrap interval. + Grafana Related Commands ------------------------ -**-G path/to/grafana data-dir** Use an external directory for the Grafana database. -This flag places the Grafana data directory outside of its container and, by doing that, makes it persistent. -This is only important if you are creating your own dashboards using the Grafana GUI and wish to keep them. +**-G path/to/grafana data-dir** Use an external directory for the Grafana database. +This flag places the Grafana data directory outside of its container and, by doing that, makes it persistent. +This is only important if you are creating your own dashboards using the Grafana GUI and wish to keep them. If not used, each run of the containers will clear all of Grafana's information. **-v comma-separated versions** Each Scylla version comes with its own set of dashboards. By default, Grafana starts with the two latest versions. The ``-v`` flag allows specifying a specific version or versions. @@ -63,7 +65,7 @@ Prometheus Related Commands **-d path/to/data-dir** Use an external directory for the Prometheus data directory. This flag places the Prometheus data directory outside of its container and by doing that makes it persistent. -.. note:: Specifying an external directory is important for systems in production. Without it, +.. note:: Specifying an external directory is important for systems in production. Without it, every restart of the monitoring stack will result in metrics lost. **-p prometheus-port** Override the default Prometheus port, this is done using port mapping, note that port mapping does not work when using the host network. @@ -95,7 +97,7 @@ Prometheus storage configuration is covered here_. For example to set the retention time to 30 days add ``-b "--storage.tsdb.retention.time=30d"`` to the ``start-all.sh`` command -Alert Manager +Alert Manager ------------- The Alertmanager handles the alerts and takes the following parameters: diff --git a/grafana-datasource.sh b/grafana-datasource.sh index df941d92d..4f41c5322 100755 --- a/grafana-datasource.sh +++ b/grafana-datasource.sh @@ -25,6 +25,8 @@ else ;; s) BASE_DIR="grafana/stack/$OPTARG/provisioning/datasources" ;; + S) SCRAP_INTERVAL="$OPTARG" + ;; :) printf "missing argument for -%s\n" "$OPTARG" >&2 echo "$usage" >&2 exit 1 @@ -49,3 +51,7 @@ if [ -z "$SCYLLA_USER" ] || [ -z "$SCYLLA_PSSWD" ]; then else sed "s/SCYLLA_USER/$SCYLLA_USER/" grafana/datasource.psswd.scylla.yml |sed "s/SCYLLA_PSSWD/$SCYLLA_PSSWD/">$BASE_DIR/datasource.scylla.yml fi + +if [[ "$SCRAP_INTERVAL" != "" ]]; then + sed -i "s/ timeInterval: *'[[:digit:]]*.*/ timeInterval: '${SCRAP_INTERVAL}s'/g" $BASE_DIR/datasource.yaml +fi \ No newline at end of file diff --git a/grafana/alternator.template.json b/grafana/alternator.template.json index f8a2f123c..c253402bd 100644 --- a/grafana/alternator.template.json +++ b/grafana/alternator.template.json @@ -65,7 +65,7 @@ }, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", op=~\"$ops\"}[60s])>0) by (op)", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", op=~\"$ops\"}[$__rate_interval])>0) by (op)", "intervalFactor": 1, "legendFormat": "{{op}}", "metric": "", @@ -83,7 +83,7 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=~\"$ops\"}[60s])>0) by (op, le))", + "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=~\"$ops\"}[$__rate_interval])>0) by (op, le))", "intervalFactor": 1, "legendFormat": "{{op}}", @@ -102,7 +102,7 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=~\"$ops\"}[60s])>0) by (op, le))", + "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=~\"$ops\"}[$__rate_interval])>0) by (op, le))", "intervalFactor": 1, "legendFormat": "{{op}}", @@ -204,7 +204,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetItem\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetItem\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -218,7 +218,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"PutItem\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"PutItem\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -232,7 +232,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"UpdateItem\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"UpdateItem\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -246,7 +246,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteItem\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteItem\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -260,7 +260,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchWriteItem\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchWriteItem\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -274,7 +274,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchGetItem\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchGetItem\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -288,7 +288,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Query\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Query\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -302,7 +302,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Scan\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Scan\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -348,7 +348,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_alternator_op_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]]) or on () $func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_op_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[$__rate_interval])) by ([[by]]) or on () $func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -362,7 +362,7 @@ "span": 3, "targets": [ { - "expr": "scylla_alternator_op_latency_summary{quantile=\"0.5\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"} or on () $func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]]) + 1)", + "expr": "scylla_alternator_op_latency_summary{quantile=\"0.5\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"} or on () $func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[$__rate_interval])) by ([[by]]) + 1)", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -376,7 +376,7 @@ "span": 3, "targets": [ { - "expr": "scylla_alternator_op_latency_summary{quantile=\"0.95\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"} or on() histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", + "expr": "scylla_alternator_op_latency_summary{quantile=\"0.95\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"} or on() histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[$__rate_interval])) by ([[by]], le))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -391,7 +391,7 @@ "span": 3, "targets": [ { - "expr": "scylla_alternator_op_latency_summary{quantile=\"0.99\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"} or on() histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", + "expr": "scylla_alternator_op_latency_summary{quantile=\"0.99\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"} or on() histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[$__rate_interval])) by ([[by]], le))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -430,7 +430,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListStreams\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListStreams\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -444,7 +444,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeStream\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeStream\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -458,7 +458,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetShardIterator\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetShardIterator\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -472,7 +472,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetRecords\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetRecords\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -518,7 +518,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_alternator_op_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]]) or on() $func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_op_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[$__rate_interval])) by ([[by]]) or on() $func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -532,7 +532,7 @@ "span": 3, "targets": [ { - "expr": "scylla_alternator_op_latency_summary{quantile=\"0.5\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"} or on() $func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]]) + 1)", + "expr": "scylla_alternator_op_latency_summary{quantile=\"0.5\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"} or on() $func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[$__rate_interval])) by ([[by]]) + 1)", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -546,7 +546,7 @@ "span": 3, "targets": [ { - "expr": "scylla_alternator_op_latency_summary{quantile=\"0.95\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"} or on () histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", + "expr": "scylla_alternator_op_latency_summary{quantile=\"0.95\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"} or on () histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[$__rate_interval])) by ([[by]], le))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -561,7 +561,7 @@ "span": 3, "targets": [ { - "expr": "scylla_alternator_op_latency_summary{quantile=\"0.99\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"} or on () histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", + "expr": "scylla_alternator_op_latency_summary{quantile=\"0.99\", by=\"$by\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"} or on () histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[$__rate_interval])) by ([[by]], le))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -600,7 +600,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"CreateTable\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"CreateTable\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -614,7 +614,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteTable\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteTable\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -628,7 +628,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeTable\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeTable\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -642,7 +642,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListTables\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListTables\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -656,7 +656,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeEndpoints\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeEndpoints\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -680,7 +680,7 @@ "span": 4, "targets": [ { - "expr": "$func(rate(scylla_expiration_items_deleted{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_expiration_items_deleted{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -720,7 +720,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -735,7 +735,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -750,7 +750,7 @@ "span": 3, "targets": [ { - "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -765,7 +765,7 @@ "span": 3, "targets": [ { - "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", diff --git a/grafana/datasource.yml b/grafana/datasource.yml index 16d8a1033..2724b0bac 100644 --- a/grafana/datasource.yml +++ b/grafana/datasource.yml @@ -7,6 +7,8 @@ datasources: access: proxy basicAuth: false isDefault: true + jsonData: + timeInterval: '20s' - name: alertmanager type: alertmanager orgId: 1 diff --git a/grafana/scylla-advanced.template.json b/grafana/scylla-advanced.template.json index b39bf80b3..90c744699 100644 --- a/grafana/scylla-advanced.template.json +++ b/grafana/scylla-advanced.template.json @@ -37,7 +37,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_io_queue_consumption{iogroup=~\"$iogroup\", class=~\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\"}[1m])) by (instance, iogroup, stream)", + "expr": "$func(rate(scylla_io_queue_consumption{iogroup=~\"$iogroup\", class=~\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\"}[$__rate_interval])) by (instance, iogroup, stream)", "intervalFactor": 1, "legendFormat": "Group {{iogroup}} {{stream}} {{dc}} {{instance}} ", "refId": "A", @@ -91,7 +91,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]]))", + "expr": "topk([[topk]], max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -127,7 +127,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -145,7 +145,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -163,7 +163,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]]))", + "expr": "topk([[topk]], max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]]))", "intervalFactor": 1, "refId": "A", "step": 30 @@ -197,7 +197,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_io_queue_starvation_time_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_io_queue_starvation_time_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_io_queue_starvation_time_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_io_queue_starvation_time_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "refId": "A", "step": 30 @@ -260,7 +260,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[$__rate_interval])/1000) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[$__rate_interval])/1000) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -279,7 +279,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[$__rate_interval])/1000) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[$__rate_interval])/1000) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -298,7 +298,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_scheduler_starvetime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_scheduler_starvetime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_scheduler_starvetime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[$__rate_interval])/1000) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_scheduler_starvetime_ms{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\",group=~\"$sg\", shard=~\"[[shard]]\"}[$__rate_interval])/1000) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -362,7 +362,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -380,7 +380,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -411,7 +411,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -429,7 +429,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -447,7 +447,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -471,7 +471,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -489,7 +489,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -507,7 +507,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -589,7 +589,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_commitlog_flush{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_commitlog_flush{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_commitlog_flush{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_commitlog_flush{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -625,7 +625,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], avg(rate(scylla_commitlog_flush_limit_exceeded{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_commitlog_flush_limit_exceeded{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], avg(rate(scylla_commitlog_flush_limit_exceeded{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], avg(rate(scylla_commitlog_flush_limit_exceeded{instance=~\"[[node]]\" ,cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -748,7 +748,7 @@ "span": 2, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_rpc_client_sent_messages{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_sent_messages{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]],domain)>0)", + "expr": "topk([[topk]], sum(rate(scylla_rpc_client_sent_messages{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_sent_messages{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]],domain)>0)", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -764,7 +764,7 @@ "span": 2, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_rpc_client_replied{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_replied{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]],domain)>0)", + "expr": "topk([[topk]], sum(rate(scylla_rpc_client_replied{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_replied{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]],domain)>0)", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -780,7 +780,7 @@ "span": 2, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_rpc_client_exception_received{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_exception_received{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]],domain)>0)", + "expr": "topk([[topk]], sum(rate(scylla_rpc_client_exception_received{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_exception_received{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]],domain)>0)", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -796,7 +796,7 @@ "span": 2, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_rpc_client_timeout{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_timeout{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]],domain)>0)", + "expr": "topk([[topk]], sum(rate(scylla_rpc_client_timeout{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain)) or on ([[by]],domain) bottomk([[bottomk]], sum(rate(scylla_rpc_client_timeout{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]],domain)>0)", "intervalFactor": 1, "legendFormat": "", "metric": "", diff --git a/grafana/scylla-cql.template.json b/grafana/scylla-cql.template.json index 2c9b8a225..728396051 100644 --- a/grafana/scylla-cql.template.json +++ b/grafana/scylla-cql.template.json @@ -49,7 +49,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -66,7 +66,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -82,7 +82,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -99,7 +99,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -134,7 +134,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_connections{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_connections{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_connections{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_connections{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -151,7 +151,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -166,7 +166,7 @@ "pointradius": 1, "targets": [ { - "expr": "sum(rate(scylla_transport_cql_errors_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]],type) >0", + "expr": "sum(rate(scylla_transport_cql_errors_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]],type) >0", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -183,7 +183,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_batches_pure_logged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_batches_pure_logged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_batches_pure_logged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_batches_pure_logged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -200,7 +200,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_batches_pure_unlogged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_batches_pure_unlogged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_batches_pure_unlogged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_batches_pure_unlogged{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -217,7 +217,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -234,7 +234,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -251,7 +251,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -548,7 +548,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -564,7 +564,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -580,7 +580,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -597,7 +597,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -645,7 +645,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -661,7 +661,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -677,7 +677,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -694,7 +694,7 @@ "pointradius": 1, "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -774,7 +774,7 @@ "description": "All requests should be paged\n\nNon Paged request sources:\n- Client modifying the fetch size\n\nNon Paged requests require reading all the results and returning them in a single request.", "targets": [ { - "expr": "100 * ((sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "expr": "100 * ((sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])))/sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -792,7 +792,7 @@ "description": "Non-Paged requests require reading all the results and returning them in a single request", "targets": [ { - "expr": "sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -806,7 +806,7 @@ "description": "All of the requests should be Token Aware\n\nNon Token Aware requests sources:\n* Non-Prepared Stamements\n* Client not using a Token Aware load balancing policy\n\nTokenAware requests are sent to a Scylla node that is also a replica. Token Un-Aware requests require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", "targets": [ { - "expr": "clamp_max(100*((sum(rate(scylla_storage_proxy_coordinator_reads_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) or on() vector(0)) + (sum(rate(scylla_storage_proxy_coordinator_writes_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) or on() vector(0)))/((sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) + 1) or vector(1)),100) OR vector(0)", + "expr": "clamp_max(100*((sum(rate(scylla_storage_proxy_coordinator_reads_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) or on() vector(0)) + (sum(rate(scylla_storage_proxy_coordinator_writes_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) or on() vector(0)))/((sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) + 1) or vector(1)),100) OR vector(0)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -822,7 +822,7 @@ "span": 3, "targets": [ { - "expr": "(sum(rate(scylla_storage_proxy_coordinator_reads_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() sum(scylla_reactor_utilization*0) by([[by]])) + (sum(rate(scylla_storage_proxy_coordinator_writes_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() sum(scylla_reactor_utilization*0) by([[by]]))", + "expr": "(sum(rate(scylla_storage_proxy_coordinator_reads_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on() sum(scylla_reactor_utilization*0) by([[by]])) + (sum(rate(scylla_storage_proxy_coordinator_writes_coordinator_outside_replica_set{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on() sum(scylla_reactor_utilization*0) by([[by]]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -842,7 +842,7 @@ "description": "Reversed CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with ORDER BY that is different from the \"CLUSTERING ORDER BY\" of the table\nAlternatives:\n\n* Denormalize your data (use a Materialized View)", "targets": [ { - "expr": "100 * sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "expr": "100 * sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -860,7 +860,7 @@ "description": "Reversed CQL Reads entail additional processing on server side and should be avoided", "targets": [ { - "expr": "sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -874,7 +874,7 @@ "description": "ALLOW FILTERING CQL Reads, the percentage of read requests with 'ALLOW FILTERING'\n\nALLOW FILTERING CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned - check \n\"ALLOW FILTERING CQL Read Filtered Rows to check what percentage of the data is used\"\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", "targets": [ { - "expr": "100 * sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "expr": "100 * sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -890,7 +890,7 @@ "description": "Read requests with ALLOW FILTERING\n\nALLOW FILTERING CQL Reads entail additional processing on server side and should be avoided", "targets": [ { - "expr": "sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -904,7 +904,7 @@ "description": "ALLOW FILTERING Filtered rows, the percentage of rows that were read and then filtered.\n\nALLOW FILTERING CQL Reads entail additional processing on server side. \nReading a row and then filter it is a waste of resources.\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", "targets": [ { - "expr": "100 * sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) /sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "expr": "100 * sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) /sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -922,7 +922,7 @@ "description": "CQL Queries with ALLOW FILTERING should be avoided.\nDropped rows are rows that were read but were filtered by the server.\nWhen dropped rows is relatively high you should consider the alternatives", "targets": [ { - "expr": "sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -930,7 +930,7 @@ "refId": "A" }, { - "expr": "sum(rate(scylla_cql_filtered_rows_matched_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_filtered_rows_matched_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "interval": "", @@ -939,7 +939,7 @@ "refId": "B" }, { - "expr": "sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "intervalFactor": 2, "legendFormat": "rows dropped $node $shard", @@ -959,7 +959,7 @@ "description": "Range scans should typically by pass the cache.\n\n Add BYPASS CACHE to your select queries.", "targets": [ { - "expr": "floor(100 *sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))/(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) )) OR vector(0)", + "expr": "floor(100 *sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))/(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) )) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -977,7 +977,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -991,7 +991,7 @@ "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", "targets": [ { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -1009,7 +1009,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1023,7 +1023,7 @@ "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", "targets": [ { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -1041,7 +1041,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1077,7 +1077,7 @@ "description": "How many Queries use Consistency level ONE\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", "targets": [ { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -1095,7 +1095,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1109,7 +1109,7 @@ "description": "How many Queries use Consistency level QUORUM\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", "targets": [ { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", "format": "time_series", "hide": false, "instant": false, @@ -1127,7 +1127,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[1m])) by ([[by]])", + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[$__rate_interval])) by ([[by]])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1149,7 +1149,7 @@ "description": "Cross DC traffic may cause additional latencies and network loads and in most cases, should be avoided.\n\nCross DC Read requests sources:\n- Consistency Level that is not LOCAL_XXX\n- Tables with read_repair_chance > 0\n\nNote:\n- If requests are supposed to be DC local - verify client is using a DCAware policy and a LOCAL_XX consistency level", "targets": [ { - "expr": "100*(sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) - sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", datacenter=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "expr": "100*(sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) - sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", datacenter=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])))/sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", "format": "time_series", "intervalFactor": 1, "refId": "A" diff --git a/grafana/scylla-detailed.template.json b/grafana/scylla-detailed.template.json index 0188cd7fb..6c861fa10 100644 --- a/grafana/scylla-detailed.template.json +++ b/grafana/scylla-detailed.template.json @@ -30,7 +30,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -46,7 +46,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -61,7 +61,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -103,7 +103,7 @@ }, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -121,7 +121,7 @@ }, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -138,7 +138,7 @@ }, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -317,7 +317,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -332,7 +332,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -347,7 +347,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_speculative_digest_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_speculative_digest_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_speculative_digest_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_speculative_digest_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -362,7 +362,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_speculative_data_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_speculative_data_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_speculative_data_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_speculative_data_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -377,7 +377,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_transport_requests_shed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_transport_requests_shed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_transport_requests_shed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_transport_requests_shed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -407,7 +407,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name) or on([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name)) or on ([[by]], scheduling_group_name) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name) or on([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name) or on([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name)) or on ([[by]], scheduling_group_name) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name) or on([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -469,7 +469,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name) or on ([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name)) or on ([[by]],scheduling_group_name) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name) or on ([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]],scheduling_group_name))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name) or on ([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name)) or on ([[by]],scheduling_group_name) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name) or on ([[by]],scheduling_group_name) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]],scheduling_group_name))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -557,7 +557,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -572,7 +572,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -587,21 +587,21 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "Read {{dc}} {{instance}} {{shard}}", "refId": "A", "step": 1 }, { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "CAS {{dc}} {{instance}} {{shard}}", "refId": "B", "step": 1 }, { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "Range {{dc}} {{instance}} {{shard}}", "refId": "C", @@ -616,7 +616,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -731,7 +731,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], clamp_max(1 + sum((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) by ([[by]])/(sum(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 0.00001),100)) or on ([[by]]) bottomk([[bottomk]], clamp_max(1 + sum((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) by ([[by]])/(sum(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 0.00001),100))", + "expr": "topk([[topk]], clamp_max(1 + sum((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) by ([[by]])/(sum(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + 0.00001),100)) or on ([[by]]) bottomk([[bottomk]], clamp_max(1 + sum((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) by ([[by]])/(sum(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + 0.00001),100))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -746,7 +746,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -761,7 +761,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -776,7 +776,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -799,7 +799,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_rpc_compression_compressed_bytes_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]], algorithm)) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_rpc_compression_compressed_bytes_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]], algorithm))", + "expr": "topk([[topk]], $func(rate(scylla_rpc_compression_compressed_bytes_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]], algorithm)) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_rpc_compression_compressed_bytes_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]], algorithm))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -815,7 +815,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_rpc_compression_compressed_bytes_received{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]], algorithm)) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_rpc_compression_compressed_bytes_received{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]], algorithm))", + "expr": "topk([[topk]], $func(rate(scylla_rpc_compression_compressed_bytes_received{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]], algorithm)) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_rpc_compression_compressed_bytes_received{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]], algorithm))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -830,7 +830,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -845,7 +845,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -947,7 +947,7 @@ "span": 6, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -962,7 +962,7 @@ "span": 6, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -982,7 +982,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -997,7 +997,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1012,7 +1012,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1027,7 +1027,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1048,7 +1048,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1063,7 +1063,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1078,7 +1078,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1093,7 +1093,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1113,7 +1113,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1128,7 +1128,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1143,7 +1143,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1158,7 +1158,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1238,7 +1238,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cql_prepared_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cql_prepared_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cql_prepared_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cql_prepared_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1253,7 +1253,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cql_authorized_prepared_statements_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cql_authorized_prepared_statements_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cql_authorized_prepared_statements_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cql_authorized_prepared_statements_cache_evictions{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1297,7 +1297,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1312,7 +1312,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1342,7 +1342,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1361,7 +1361,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1406,7 +1406,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_sstables_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_sstables_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1421,7 +1421,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_range_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1436,7 +1436,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_sstables_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_sstables_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1451,7 +1451,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cache_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cache_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cache_row_tombstone_reads{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1466,7 +1466,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_sstables_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_sstables_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1481,7 +1481,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_sstables_range_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_range_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_sstables_range_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_range_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1496,7 +1496,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_sstables_cell_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_cell_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_sstables_cell_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_sstables_cell_tombstone_writes{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1539,7 +1539,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1554,7 +1554,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(casrlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]) + 1)) or on ([[by]]) bottomk([[bottomk]], sum(casrlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]) + 1))", + "expr": "topk([[topk]], sum(casrlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) + 1)) or on ([[by]]) bottomk([[bottomk]], sum(casrlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) + 1))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1584,7 +1584,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1604,7 +1604,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on ([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on ([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1619,7 +1619,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], sum(caswlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 1))) or on ([[by]]) bottomk([[bottomk]], sum(caswlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 1)))", + "expr": "topk([[topk]], sum(caswlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + 1))) or on ([[by]]) bottomk([[bottomk]], sum(caswlatencya{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"}) or on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_sum{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + 1)))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1649,7 +1649,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1669,7 +1669,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1726,7 +1726,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_condition_not_met{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_condition_not_met{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_condition_not_met{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_condition_not_met{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1741,7 +1741,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1756,7 +1756,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]) - $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_bucket{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", le=\"1.000000\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]) - $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_bucket{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", le=\"1.000000\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) - $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_bucket{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", le=\"1.000000\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) - $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_bucket{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", le=\"1.000000\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1771,7 +1771,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeout_due_to_uncertainty{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeout_due_to_uncertainty{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeout_due_to_uncertainty{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_timeout_due_to_uncertainty{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1791,7 +1791,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1806,7 +1806,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unavailable{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1821,7 +1821,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_write_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1836,7 +1836,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_read_unfinished_commit{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1856,7 +1856,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_failed_read_round_optimization{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_failed_read_round_optimization{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_failed_read_round_optimization{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_failed_read_round_optimization{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1871,7 +1871,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1886,7 +1886,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_dropped_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_dropped_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_cas_dropped_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_cas_dropped_prune{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1929,7 +1929,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cdc_operations_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cdc_operations_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_cdc_operations_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cdc_operations_total{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -1944,7 +1944,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_cdc_operations_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 1)) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cdc_operations_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 1))", + "expr": "topk([[topk]], $func(rate(scylla_cdc_operations_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + 1)) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_cdc_operations_failed{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + 1))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -2075,7 +2075,7 @@ "span": 4, "targets": [ { - "expr": "topk([[topk]], ($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[1m])) by ([[by]]))/10) or on ([[by]]) bottomk([[bottomk]], ($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[1m])) by ([[by]]))/10)", + "expr": "topk([[topk]], ($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[$__rate_interval])) by ([[by]]))/10) or on ([[by]]) bottomk([[bottomk]], ($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[$__rate_interval])) by ([[by]]))/10)", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -2123,7 +2123,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -2182,7 +2182,7 @@ "span": 3, "targets": [ { - "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[60s])) by ([[by]]))", + "expr": "topk([[topk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -2243,7 +2243,7 @@ "description": "Bytes received in CQL messages", "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -2260,7 +2260,7 @@ "description": "Average CQL message size (received)", "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\", instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\", instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\", instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"$kind\", instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -2277,7 +2277,7 @@ "description": "Bytes sent in CQL messages", "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -2293,7 +2293,7 @@ "dashversion":[">5.3", ">2022.1"], "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/sum(rate(scylla_transport_cql_requests_count{kind=~\"$kind\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -2310,7 +2310,7 @@ "dashversion":[">5.3", ">2022.1"], "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]) or on([[by]]) sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]))) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]) or on([[by]]) sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on([[by]]) sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_request_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on([[by]]) sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])))", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -2327,7 +2327,7 @@ "dashversion":[">5.3", ">2022.1"], "targets": [ { - "expr": "topk([[topk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]) or on ([[by]]) sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]))) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]) or on ([[by]]) sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])))", + "expr": "topk([[topk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]))) or on ([[by]]) bottomk([[bottomk]], sum(rate(scylla_transport_cql_response_bytes{kind=~\"QUERY|EXECUTE\",instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on ([[by]]) sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])))", "intervalFactor": 1, "legendFormat": "", "metric": "", diff --git a/grafana/scylla-ks.template.json b/grafana/scylla-ks.template.json index 83f104fce..02308fab4 100644 --- a/grafana/scylla-ks.template.json +++ b/grafana/scylla-ks.template.json @@ -40,7 +40,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_column_family_write_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_column_family_write_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -98,7 +98,7 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_column_family_read_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "expr": "$func(rate(scylla_column_family_read_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "", "refId": "A", diff --git a/grafana/scylla-manager.dbaas.template.json b/grafana/scylla-manager.dbaas.template.json index 382ceb66c..1bb956339 100644 --- a/grafana/scylla-manager.dbaas.template.json +++ b/grafana/scylla-manager.dbaas.template.json @@ -186,7 +186,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(node_cpu_seconds_total{cluster=~\"$cluster|$^\", mode=\"idle\"}[60s])) by (cluster,instance)", + "expr": "sum(rate(node_cpu_seconds_total{cluster=~\"$cluster|$^\", mode=\"idle\"}[$__rate_interval])) by (cluster,instance)", "intervalFactor": 1, "legendFormat": "", "metric": "", @@ -216,7 +216,7 @@ "span": 3, "targets": [ { - "expr": "sum(rate(node_network_transmit_bytes{cluster=~\"$cluster|$^\", device!~\"lo\"}[60s])) by (cluster,instance)", + "expr": "sum(rate(node_network_transmit_bytes{cluster=~\"$cluster|$^\", device!~\"lo\"}[$__rate_interval])) by (cluster,instance)", "intervalFactor": 1, "legendFormat": "", "metric": "", diff --git a/grafana/scylla-manager.template.json b/grafana/scylla-manager.template.json index 41a8cda38..2a5ad50b4 100644 --- a/grafana/scylla-manager.template.json +++ b/grafana/scylla-manager.template.json @@ -368,7 +368,7 @@ }, "targets": [ { - "expr": "sum(rate(scylla_manager_repair_token_ranges_success{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}[60s])) by ([[by]])", + "expr": "sum(rate(scylla_manager_repair_token_ranges_success{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "refId": "A", "step": 4 @@ -387,7 +387,7 @@ }, "targets": [ { - "expr": "sum(rate(scylla_manager_repair_token_ranges_error{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}[60s])) by ([[by]])", + "expr": "sum(rate(scylla_manager_repair_token_ranges_error{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "refId": "A", "step": 4 diff --git a/grafana/scylla-overview.template.json b/grafana/scylla-overview.template.json index ab0be2f57..88977504c 100644 --- a/grafana/scylla-overview.template.json +++ b/grafana/scylla-overview.template.json @@ -37,7 +37,7 @@ "span": 2, "targets": [ { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))", + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))", "intervalFactor": 1, "legendFormat": "", "refId": "A", @@ -76,7 +76,7 @@ "span": 2, "targets": [ { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))", + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))", "intervalFactor": 1, "legendFormat": "Reads", "refId": "A", @@ -235,14 +235,14 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "Hit {{instance}} {{shard}}", "refId": "A", "step": 10 }, { - "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "Misses {{instance}} {{shard}}", "refId": "B", @@ -266,14 +266,14 @@ "span": 3, "targets": [ { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "Writes {{instance}} {{shard}}", "refId": "A", "step": 1 }, { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1d)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1d))", + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval] offset 1d)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval] offset 1d))", "legendFormat": "1 Day Ago", "interval": "", "intervalFactor": 1, @@ -281,7 +281,7 @@ "step": 1 }, { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1w)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1w))", + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval] offset 1w)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval] offset 1w))", "legendFormat": "1 Week Ago", "interval": "", "intervalFactor": 1, @@ -337,7 +337,7 @@ "span": 2, "targets": [ { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "Writes {{instance}} {{shard}}", "refId": "A", @@ -357,21 +357,21 @@ }, "targets": [ { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])", + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval])) by ([[by]])", "intervalFactor": 1, "legendFormat": "Reads {{instance}} {{shard}}", "refId": "A", "step": 1 }, { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1d)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1d))", + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval] offset 1d)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval] offset 1d))", "intervalFactor": 1, "legendFormat": "1 Day Ago", "refId": "B", "step": 1 }, { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1w)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1w))", + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval] offset 1w)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[$__rate_interval] offset 1w))", "intervalFactor": 1, "legendFormat": "1 Week Ago", "refId": "C", @@ -432,7 +432,7 @@ }, "targets": [ { - "expr": "($func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or vector(0))+on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or vector(0))+on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or vector(0))", + "expr": "($func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or vector(0))+on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or vector(0))+on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or vector(0))", "intervalFactor": 1, "legendFormat": "Read {{instance}} {{shard}}", "refId": "A", diff --git a/grafana/types.json b/grafana/types.json index 6eff83e46..4f0f31bb8 100644 --- a/grafana/types.json +++ b/grafana/types.json @@ -588,7 +588,7 @@ "class":"small_stat", "targets":[ { - "expr":"sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", + "expr":"sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) or on() vector(0))", "intervalFactor":1, "refId":"A", "instant":true, @@ -683,7 +683,7 @@ }, "targets":[ { - "expr":"(sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) or on() vector(0)) + (sum(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) or on() vector(0))", + "expr":"(sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) or on() vector(0)) + (sum(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) or on() vector(0))", "intervalFactor":1, "refId":"A", "instant":true, @@ -1018,7 +1018,7 @@ }, "targets":[ { - "expr":"sum(rate(scylla_alternator_op_latency_sum{cluster=~\"$cluster\", op=~\"$ops\"}[1m])) by (op)/sum(rate(scylla_alternator_op_latency_count{cluster=~\"$cluster|^$\", op=~\"$ops\"}[1m])>0) by (op)", + "expr":"sum(rate(scylla_alternator_op_latency_sum{cluster=~\"$cluster\", op=~\"$ops\"}[$__rate_interval])) by (op)/sum(rate(scylla_alternator_op_latency_count{cluster=~\"$cluster|^$\", op=~\"$ops\"}[$__rate_interval])>0) by (op)", "intervalFactor":1, "legendFormat": "{{op}}", "refId":"A", @@ -1060,7 +1060,7 @@ }, "targets":[ { - "expr":"histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=~\"$ops\"}[60s])>0) by (op, le))", + "expr":"histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=~\"$ops\"}[$__rate_interval])>0) by (op, le))", "intervalFactor":1, "legendFormat": "{{op}}", "refId":"A", @@ -1074,7 +1074,7 @@ "class":"small_stat", "targets":[ { - "expr":"sum(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]))", + "expr":"sum(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))", "intervalFactor":1, "refId":"A", "instant":true, @@ -1170,7 +1170,7 @@ }, "targets":[ { - "expr":"(sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) or on() vector(0)) + (sum(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) or on() vector(0))", + "expr":"(sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) or on() vector(0)) + (sum(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) or on() vector(0))", "intervalFactor":1, "refId":"A", "instant":true, @@ -2769,7 +2769,7 @@ "format":"table" }, { - "expr":"sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", + "expr":"sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[$__rate_interval])) by (instance)", "legendFormat":"", "interval":"", "refId":"C", @@ -2785,7 +2785,7 @@ "format":"table" }, { - "expr":"(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", + "expr":"(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[$__rate_interval])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", "legendFormat":"", "interval":"", "refId":"E", @@ -2827,7 +2827,7 @@ "format":"table" }, { - "expr":"100*sum(rate(scylla_cache_row_misses{dc=~\"$dc\"}[1m])) by (instance)/(sum(rate(scylla_cache_row_hits{dc=~\"$dc\"}[1m])) by (instance) + sum(rate(scylla_cache_row_misses{dc=~\"$dc\"}[1m])) by (instance))", + "expr":"100*sum(rate(scylla_cache_row_misses{dc=~\"$dc\"}[$__rate_interval])) by (instance)/(sum(rate(scylla_cache_row_hits{dc=~\"$dc\"}[$__rate_interval])) by (instance) + sum(rate(scylla_cache_row_misses{dc=~\"$dc\"}[$__rate_interval])) by (instance))", "legendFormat":"", "interval":"", "refId":"J", @@ -2835,7 +2835,7 @@ "format":"table" }, { - "expr":"histogram_quantile(0.5, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])>0) by (le, instance))", + "expr":"histogram_quantile(0.5, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}[$__rate_interval])>0) by (le, instance))", "legendFormat":"", "interval":"", "refId":"K", @@ -2843,7 +2843,7 @@ "format":"table" }, { - "expr":"histogram_quantile(0.9, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])>0) by (le, instance))", + "expr":"histogram_quantile(0.9, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}[$__rate_interval])>0) by (le, instance))", "legendFormat":"", "interval":"", "refId":"L", @@ -2851,7 +2851,7 @@ "format":"table" }, { - "expr":"histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])>0) by (le, instance))", + "expr":"histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}[$__rate_interval])>0) by (le, instance))", "legendFormat":"", "interval":"", "refId":"M", @@ -3679,7 +3679,7 @@ "format":"table" }, { - "expr":"sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", + "expr":"sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[$__rate_interval])) by (instance)", "legendFormat":"", "interval":"", "refId":"C", @@ -3695,7 +3695,7 @@ "format":"table" }, { - "expr":"(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", + "expr":"(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[$__rate_interval])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", "legendFormat":"", "interval":"", "refId":"E", diff --git a/prometheus-config.sh b/prometheus-config.sh index 897ca2c5e..e38b618ba 100755 --- a/prometheus-config.sh +++ b/prometheus-config.sh @@ -13,29 +13,48 @@ if [ "$1" = "" ]; then fi for arg; do shift - case $arg in - (--compose) COMPOSE=1 - AM_ADDRESS="aalert:9093" - ;; - (--no-cas-cdc) - NO_CAS="1" - NO_CDC="1" - ;; - (--no-cas) - NO_CAS="1" - ;; - (--no-cdc) - NO_CDC="1" - ;; - (--no-node-exporter-file) - NO_NODE_EXPORTER_FILE="1" - ;; - (--no-manager-agent-file) - NO_MANAGER_AGENT_FILE="1" - ;; - (*) set -- "$@" "$arg" - ;; - esac + if [ -z "$PARAM" ]; then + case $arg in + (--compose) COMPOSE=1 + AM_ADDRESS="aalert:9093" + ;; + (--no-cas-cdc) + NO_CAS="1" + NO_CDC="1" + ;; + (--no-cas) + NO_CAS="1" + ;; + (--no-cdc) + NO_CDC="1" + ;; + (--scrap) + PARAM="scrap" + ;; + (--no-node-exporter-file) + NO_NODE_EXPORTER_FILE="1" + ;; + (--no-manager-agent-file) + NO_MANAGER_AGENT_FILE="1" + ;; + (*) set -- "$@" "$arg" + ;; + esac + else + DOCR=`echo $arg|cut -d',' -f1` + VALUE=`echo $arg|cut -d',' -f2-|sed 's/#/ /g'` + NOSPACE=`echo $arg|sed 's/ /#/g'` + if [[ $NOSPACE == --* ]]; then + echo "Error: No value given to --$PARAM" + echo + usage + exit 1 + fi + if [ "$PARAM" = "scrap" ]; then + SCRAP_INTERVAL="$NOSPACE" + fi + unset PARAM + fi done while getopts ':hL:m:T:E:s:' option; do @@ -77,6 +96,11 @@ fi if [[ "$EVALUATION_INTERVAL" != "" ]]; then sed -i "s/ evaluation_interval: [[:digit:]]*.*/ evaluation_interval: ${EVALUATION_INTERVAL}/g" $BASE_DIR/prometheus.yml fi +if [[ "$SCRAP_INTERVAL" != "" ]]; then + sed -i "s/ scrape_interval: [[:digit:]]*.*# *Default.*/ scrape_interval: ${SCRAP_INTERVAL}s/g" $BASE_DIR/prometheus.yml + TIMEOUT=$(($SCRAP_INTERVAL - 5)) + sed -i "s/ scrape_timeout: [[:digit:]]*.*# *Default.*/ scrape_timeout: ${TIMEOUT}s/g" $BASE_DIR/prometheus.yml +fi if [ "$NO_CAS" = "1" ] && [ "$NO_CDC" = "1" ]; then sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cdc_.*|.*_cas.*)'\\n action: drop/g" $BASE_DIR/prometheus.yml elif [ "$NO_CAS" = "1" ]; then diff --git a/prometheus/prometheus.consul.yml.template b/prometheus/prometheus.consul.yml.template index 4014a6dfe..190607f97 100644 --- a/prometheus/prometheus.consul.yml.template +++ b/prometheus/prometheus.consul.yml.template @@ -1,6 +1,6 @@ global: - scrape_interval: 20s # By default, scrape targets every 20 second. - scrape_timeout: 15s # Timeout before trying to scrape a target again + scrape_interval: 20s # Default Scrape + scrape_timeout: 15s # # Default Scrape timeout evaluation_interval: 20s # Attach these labels to any time series or alerts when communicating with diff --git a/prometheus/prometheus.yml.template b/prometheus/prometheus.yml.template index eeed516b9..22871fce1 100644 --- a/prometheus/prometheus.yml.template +++ b/prometheus/prometheus.yml.template @@ -1,6 +1,6 @@ global: - scrape_interval: 20s # By default, scrape targets every 20 second. - scrape_timeout: 15s # Timeout before trying to scrape a target again + scrape_interval: 20s # Default Scrape + scrape_timeout: 15s # Default Scrape timeout evaluation_interval: 20s # Attach these labels to any time series or alerts when communicating with diff --git a/start-all.sh b/start-all.sh index 014874b0b..e6212da1f 100755 --- a/start-all.sh +++ b/start-all.sh @@ -97,6 +97,7 @@ Options: --thanos-sc - If set, run thanos side car with the Prometheus server. --thanos - If set, run thanos query as a Grafana datasource. --enable-protobuf - If set, enable the experimental Prometheus Protobuf with Native histograms support. + --scrap [scrap duration] - Change the default Prometheus scrap duration. Duration is in seconds. --target-directory - If set, prometheus/targets/ directory will be set as a root directory for the target files the file names should be scylla_server.yml, node_exporter_servers.yml, and scylla_manager_servers.yml --stack id - Use this option when running a secondary stack, id could be 1-4 @@ -278,6 +279,10 @@ for arg; do LIMIT="1" PARAM="stack" ;; + (--scrap) + LIMIT="1" + PARAM="scrap" + ;; (--no-cas-cdc) PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-cas-cdc" ;; @@ -347,6 +352,9 @@ for arg; do STACK_CMD="-s $NOSPACE" STACK="/stack/$NOSPACE" unset PARAM + elif [ "$PARAM" = "scrap" ]; then + SCRAP_CMD="--scrap $NOSPACE" + unset PARAM elif [ "$PARAM" = "archive" ]; then DATA_DIR="$NOSPACE" unset PARAM @@ -669,7 +677,7 @@ for val in "${PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY[@]}"; do fi done -./prometheus-config.sh -m $AM_ADDRESS $STACK_CMD $CONSUL_ADDRESS $PROMETHEUS_TARGETS +./prometheus-config.sh -m $AM_ADDRESS $STACK_CMD $SCRAP_CMD $CONSUL_ADDRESS $PROMETHEUS_TARGETS if [ "$DATA_DIR" != "" ] && [ "$ARCHIVE" != "1" ]; then DATE=$(date +"%Y-%m-%d_%H_%M_%S") if [ -f $DATA_DIR/scylla.txt ]; then @@ -791,4 +799,4 @@ fi if [ "$RUN_ALTERNATOR" = 1 ]; then GRAFANA_ENV_COMMAND="$GRAFANA_ENV_COMMAND --alternator" fi -./start-grafana.sh $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS $GRAFANA_ENV_COMMAND $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD $STACK_CMD +./start-grafana.sh $SCRAP_CMD $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS $GRAFANA_ENV_COMMAND $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD $STACK_CMD diff --git a/start-grafana.sh b/start-grafana.sh index 3a0d9ed0b..f6bf632a8 100755 --- a/start-grafana.sh +++ b/start-grafana.sh @@ -60,6 +60,10 @@ for arg; do LIMIT="1" PARAM="1" ;; + (--scrap) + PARAM="scrap" + LIMIT="1" + ;; (--auth) GRAFANA_AUTH=true ;; @@ -80,6 +84,9 @@ for arg; do DOCKER_PARAMS[$DOCR]="${DOCKER_PARAMS[$DOCR]} $VALUE" PARAMS="$PARAMS --param $NOSPACE" unset PARAM + elif [ "$PARAM" = "scrap" ]; then + SCRAP_INTERVAL="-S $NOSPACE" + unset PARAM else if [ -z "${DOCKER_LIMITS[$DOCR]}" ]; then DOCKER_LIMITS[$DOCR]="" @@ -249,7 +256,7 @@ for val in "${GRAFANA_DASHBOARD_ARRAY[@]}"; do GRAFANA_DASHBOARD_COMMAND="$GRAFANA_DASHBOARD_COMMAND -j $val" done ./generate-dashboards.sh -t $SPECIFIC_SOLUTION -v $VERSIONS -M $MANAGER_VERSION $STACK_CMD $GRAFANA_DASHBOARD_COMMAND -./grafana-datasource.sh $DATA_SOURCES $STACK_CMD +./grafana-datasource.sh $DATA_SOURCES $STACK_CMD $SCRAP_INTERVAL if [[ ! $DOCKER_PARAM = *"--net=host"* ]]; then PORT_MAPPING="-p $BIND_ADDRESS$GRAFANA_PORT:3000"