From 1b6fb3317ef99d60ebad7f5a1f7fb7116e50025c Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Fri, 22 Jul 2022 15:00:17 -0700 Subject: [PATCH 1/2] Add a dashboard utils file Signed-off-by: Callum Styan --- dashboard-utils/utils.libsonnet | 283 ++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 dashboard-utils/utils.libsonnet diff --git a/dashboard-utils/utils.libsonnet b/dashboard-utils/utils.libsonnet new file mode 100644 index 000000000..ecfe42e77 --- /dev/null +++ b/dashboard-utils/utils.libsonnet @@ -0,0 +1,283 @@ +local utils = import 'mixin-utils/utils.libsonnet'; + +(import 'grafana-builder/grafana.libsonnet') { + // Override the dashboard constructor to add: + // - default tags, + // - some links that propagate the selected cluster. + dashboard(title, uid='', service_metric, links_title):: + super.dashboard(title, uid) + { + addRowIf(condition, row):: + if condition + then self.addRow(row) + else self, + addLog(name='logs'):: self { + templating+: { + list+: [ + { + hide: 0, + label: null, + name: name, + options: [], + query: 'loki', + refresh: 1, + regex: '', + type: 'datasource', + }, + ], + }, + }, + + addCluster(multi=false):: + if multi then + self.addMultiTemplate('cluster', service_metric, $._config.per_cluster_label) + else + self.addTemplate('cluster', service_metric, $._config.per_cluster_label), + + addNamespace(multi=false):: + if multi then + self.addMultiTemplate('namespace', service_metric + '{' + $._config.per_cluster_label + '=~"$cluster"}', 'namespace') + else + self.addTemplate('namespace', service_metric + '{' + $._config.per_cluster_label + '=~"$cluster"}', 'namespace'), + + addTag():: + self + { + tags+: $._config.tags, + links+: [ + { + asDropdown: true, + icon: 'external link', + includeVars: true, + keepTime: true, + tags: $._config.tags, + targetBlank: false, + title: links_title, + type: 'dashboards', + }, + ], + }, + + addClusterSelectorTemplates(multi=true):: + local d = self { + tags: $._config.tags, + links: [ + { + asDropdown: true, + icon: 'external link', + includeVars: true, + keepTime: true, + tags: $._config.tags, + targetBlank: false, + title: 'Loki Dashboards', + type: 'dashboards', + }, + ], + }; + + if multi then + d.addMultiTemplate('cluster', service_metric, $._config.per_cluster_label) + .addMultiTemplate('namespace', service_metric + '{' + $._config.per_cluster_label + '=~"$cluster"}', 'namespace') + else + d.addTemplate('cluster', service_metric, $._config.per_cluster_label) + .addTemplate('namespace', service_metric + '{' + $._config.per_cluster_label + '=~"$cluster"}', 'namespace'), + }, + + jobMatcher(job):: + $._config.per_cluster_label + '=~"$cluster", job=~"($namespace)/%s"' % job, + + namespaceMatcher():: + $._config.per_cluster_label + '=~"$cluster", namespace=~"$namespace"', + + containerLabelMatcher(containerName):: + 'label_name=~"%s.*"' % containerName, + + logPanel(title, selector, datasource='$logs'):: { + title: title, + type: 'logs', + datasource: datasource, + targets: [ + { + refId: 'A', + expr: selector, + }, + ], + }, + fromNowPanel(title, metric_name):: + $.panel(title) + + { + type: 'stat', + title: title, + fieldConfig: { + defaults: { + custom: {}, + thresholds: { + mode: 'absolute', + steps: [ + { + color: 'green', + value: null, + }, + ], + }, + color: { + mode: 'fixed', + fixedColor: 'blue', + }, + unit: 'dateTimeFromNow', + }, + }, + targets: [ + { + expr: '%s{%s} * 1e3' % [metric_name, $.namespaceMatcher()], + refId: 'A', + instant: true, + format: 'time_series', + }, + ], + options: { + reduceOptions: { + values: false, + calcs: [ + 'lastNotNull', + ], + fields: '', + }, + orientation: 'auto', + text: {}, + textMode: 'auto', + colorMode: 'value', + graphMode: 'area', + justifyMode: 'auto', + }, + datasource: '$datasource', + }, + CPUUsagePanel(title, matcher):: + $.panel(title) + + $.queryPanel([ + 'sum by(pod) (rate(container_cpu_usage_seconds_total{%s, %s}[$__rate_interval]))' % [$.namespaceMatcher(), matcher], + 'min(container_spec_cpu_quota{%s, %s} / container_spec_cpu_period{%s, %s})' % [$.namespaceMatcher(), matcher, $.namespaceMatcher(), matcher], + ], ['{{pod}}', 'limit']) + + { + seriesOverrides: [ + { + alias: 'limit', + color: '#E02F44', + fill: 0, + }, + ], + tooltip: { sort: 2 }, // Sort descending. + }, + containerCPUUsagePanel(title, containerName):: + self.CPUUsagePanel(title, 'container="%s"' % containerName), + + memoryWorkingSetPanel(title, matcher):: + $.panel(title) + + $.queryPanel([ + // We use "max" instead of "sum" otherwise during a rolling update of a statefulset we will end up + // summing the memory of the old pod (whose metric will be stale for 5m) to the new pod. + 'max by(pod) (container_memory_working_set_bytes{%s, %s})' % [$.namespaceMatcher(), matcher], + 'min(container_spec_memory_limit_bytes{%s, %s} > 0)' % [$.namespaceMatcher(), matcher], + ], ['{{pod}}', 'limit']) + + { + seriesOverrides: [ + { + alias: 'limit', + color: '#E02F44', + fill: 0, + }, + ], + yaxes: $.yaxes('bytes'), + tooltip: { sort: 2 }, // Sort descending. + }, + containerMemoryWorkingSetPanel(title, containerName):: + self.memoryWorkingSetPanel(title, 'container="%s"' % containerName), + + goHeapInUsePanel(title, jobName):: + $.panel(title) + + $.queryPanel( + 'sum by(%s) (go_memstats_heap_inuse_bytes{%s})' % [$._config.per_instance_label, $.jobMatcher(jobName)], + '{{%s}}' % $._config.per_instance_label + ) + + { + yaxes: $.yaxes('bytes'), + tooltip: { sort: 2 }, // Sort descending. + }, + + filterNodeDisk(matcher):: + ||| + ignoring(%s) group_right() (label_replace(count by(%s, %s, device) (container_fs_writes_bytes_total{%s, %s, device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0) + ||| % [$._config.per_instance_label, $._config.per_node_label, $._config.per_instance_label, $.namespaceMatcher(), matcher], + filterNodeDiskContainer(containerName):: + self.filterNodeDisk('container="%s"' % containerName), + + newStatPanel(queries, legends='', unit='percentunit', decimals=1, thresholds=[], instant=false, novalue=''):: + super.queryPanel(queries, legends) + { + type: 'stat', + targets: [ + target { + instant: instant, + interval: '', + + // Reset defaults from queryPanel(). + format: null, + intervalFactor: null, + step: null, + } + for target in super.targets + ], + fieldConfig: { + defaults: { + decimals: decimals, + noValue: novalue, + unit: unit, + }, + overrides: [], + }, + }, + + containerDiskSpaceUtilizationPanel(title, containerName):: + $.panel(title) + + $.queryPanel('max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{%s} / kubelet_volume_stats_capacity_bytes{%s}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{%s,%s})' % [$.namespaceMatcher(), $.namespaceMatcher(), $.namespaceMatcher(), $.containerLabelMatcher(containerName)], '{{persistentvolumeclaim}}') + + { yaxes: $.yaxes('percentunit') }, + + barGauge(queries, legends='', thresholds=[], unit='short', min=null, max=null):: + super.queryPanel(queries, legends) + { + type: 'bargauge', + targets: [ + target { + // Reset defaults from queryPanel(). + format: null, + intervalFactor: null, + step: null, + } + for target in super.targets + ], + fieldConfig: { + defaults: { + color: { mode: 'thresholds' }, + mappings: [], + max: max, + min: min, + thresholds: { + mode: 'absolute', + steps: thresholds, + }, + unit: unit, + }, + }, + options: { + displayMode: 'basic', + orientation: 'horizontal', + reduceOptions: { + calcs: ['lastNotNull'], + fields: '', + values: false, + }, + }, + }, + + // Switches a panel from lines (default) to bars. + bars:: { + bars: true, + lines: false, + }, +} From ad99fbebcb099a2a4de0ebaade5179db1ec68554 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Fri, 22 Jul 2022 16:18:33 -0700 Subject: [PATCH 2/2] Fix linting Signed-off-by: Callum Styan --- dashboard-utils/utils.libsonnet | 76 ++++++++++++++++----------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/dashboard-utils/utils.libsonnet b/dashboard-utils/utils.libsonnet index ecfe42e77..16bddd2cd 100644 --- a/dashboard-utils/utils.libsonnet +++ b/dashboard-utils/utils.libsonnet @@ -240,44 +240,44 @@ local utils = import 'mixin-utils/utils.libsonnet'; { yaxes: $.yaxes('percentunit') }, barGauge(queries, legends='', thresholds=[], unit='short', min=null, max=null):: - super.queryPanel(queries, legends) + { - type: 'bargauge', - targets: [ - target { - // Reset defaults from queryPanel(). - format: null, - intervalFactor: null, - step: null, - } - for target in super.targets - ], - fieldConfig: { - defaults: { - color: { mode: 'thresholds' }, - mappings: [], - max: max, - min: min, - thresholds: { - mode: 'absolute', - steps: thresholds, - }, - unit: unit, - }, - }, - options: { - displayMode: 'basic', - orientation: 'horizontal', - reduceOptions: { - calcs: ['lastNotNull'], - fields: '', - values: false, - }, - }, + super.queryPanel(queries, legends) + { + type: 'bargauge', + targets: [ + target { + // Reset defaults from queryPanel(). + format: null, + intervalFactor: null, + step: null, + } + for target in super.targets + ], + fieldConfig: { + defaults: { + color: { mode: 'thresholds' }, + mappings: [], + max: max, + min: min, + thresholds: { + mode: 'absolute', + steps: thresholds, }, - - // Switches a panel from lines (default) to bars. - bars:: { - bars: true, - lines: false, + unit: unit, }, + }, + options: { + displayMode: 'basic', + orientation: 'horizontal', + reduceOptions: { + calcs: ['lastNotNull'], + fields: '', + values: false, + }, + }, + }, + + // Switches a panel from lines (default) to bars. + bars:: { + bars: true, + lines: false, + }, }