diff --git a/CHANGELOG.md b/CHANGELOG.md index aaea470a..771b54f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # SAS Viya Monitoring for Kubernetes +## Unreleased +* **Metrics** + * [CHANGE] New Grafana dashboard Perf/Analysis added + * [CHANGE] Server-Side Apply now used in monitoring/bin/deploy_dashboards.sh script + ## Version 1.2.24 (16APR2024) * **Metrics** * [FIX] Connect to Grafana using https from auto-provisioning sidecar containers when TLS is enabled diff --git a/bin/kube-include.sh b/bin/kube-include.sh index 09eebb53..c656f5d4 100644 --- a/bin/kube-include.sh +++ b/bin/kube-include.sh @@ -31,7 +31,7 @@ fi # 2024.01 1.25 1.27 # 2024.02 1.26 1.28 # 2024.03 1.26 1.28 -# 2024.04 1.27 1.29 +# 2024.04 1.26 1.28 if [[ $KUBE_SERVER_VER =~ v1.2[1-9] ]]; then : diff --git a/monitoring/bin/deploy_dashboards.sh b/monitoring/bin/deploy_dashboards.sh index 85114747..aafdd420 100755 --- a/monitoring/bin/deploy_dashboards.sh +++ b/monitoring/bin/deploy_dashboards.sh @@ -39,7 +39,7 @@ function deploy_dashboards { log_debug "Deploying dashboard from file [$f]" name=$(basename $f .json) - kubectl create cm -n $DASH_NS $name --dry-run=client --from-file $f -o yaml | kubectl apply -f - + kubectl create cm -n $DASH_NS $name --dry-run=client --from-file $f -o yaml | kubectl apply --server-side -f - kubectl label cm -n $DASH_NS $name --overwrite grafana_dashboard=1 sas.com/monitoring-base=kube-viya-monitoring sas.com/dashboardType=$type fi done @@ -55,7 +55,7 @@ if [ "$1" != "" ]; then f=$1 log_info "Deploying Grafana dashboard [$f]" name=$(basename $f .json) - kubectl create cm -n $DASH_NS $name --dry-run=client --from-file $f -o yaml | kubectl apply -f - + kubectl create cm -n $DASH_NS $name --dry-run=client --from-file $f -o yaml | kubectl apply --server-side -f - kubectl label cm -n $DASH_NS $name --overwrite grafana_dashboard=1 sas.com/monitoring-base=kube-viya-monitoring sas.com/dashboardType=manual exit $? else diff --git a/monitoring/dashboards/kube/perf-k8s-analysis.json b/monitoring/dashboards/kube/perf-k8s-analysis.json new file mode 100644 index 00000000..1525ac4a --- /dev/null +++ b/monitoring/dashboards/kube/perf-k8s-analysis.json @@ -0,0 +1,8341 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "uid": "$datasource" + }, + "enable": false, + "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", namespace=~\"$namespace\", pod=\"$pod\"}[2m]) > 0)", + "hide": false, + "iconColor": "rgba(215, 44, 44, 1)", + "name": "Restarts", + "showIn": 0, + "tags": [ + "restart" + ], + "type": "rows" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 53, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 83, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Bird's Eye View", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "watch the sas-readiness container to see when viya begins starting up (container exists but ready=0), and viya startup complete (ready=1)\n\nalso including container restarts in this graph\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 50, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (container,pod) (kube_pod_container_status_ready{namespace=~\"$namespace\", container=\"sas-readiness\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "VIYA_UP({{container}})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "increase(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"}[1m])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "restarts({{container}})", + "refId": "B" + } + ], + "title": "Is Viya Up?", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Pod replicas \"scheduled\" lags behind \"desired\" due to node startup time (this is normal). \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "HPA pod replicas desired (viya)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HPA pod replicas ready (viya)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 3, + "y": 1 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# HPA pod replicas desired ({{namespace}})\r\nsum by(namespace) (kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "HPA pod replicas desired ({{namespace}})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# HPA pod replicas ready ({{namespace}})\r\nsum by(namespace) (kube_deployment_status_replicas_available{\r\n namespace=~\"$namespace\"\r\n , deployment=~\"$pod\"\r\n , deployment!=\"sas-opendistro-operator\"\r\n , deployment!=\"sas-readiness\"\r\n , deployment!=\"sas-prepull\"\r\n , deployment!=\"sas-data-server-utility\"\r\n , deployment!=\"sas-connect-spawner\"\r\n , deployment!~\"sas-crunchy-data-postgres.*\"\r\n , deployment!=\"sas-connect-spawner\"\r\n , deployment!=\"sas-config-reconciler\"\r\n , deployment!=\"sas-cas-operator\"\r\n , deployment!=\"prometheus-pushgateway\"\r\n , deployment!=\"sas-data-server-operator\"\r\n})", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "HPA pod replicas ready ({{namespace}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# nodepool: {{label_agentpool}}\r\n(sum by (label_agentpool) (kube_node_labels{node=~\"$Node\", node=~\".*$NodeClass.*\"}))", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "nodepool: {{label_agentpool}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# Total \"Ready\" Nodes\r\nsum by (status) (kube_node_status_condition{condition=\"Ready\", status=\"true\"})", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total \"Ready\" Nodes", + "range": true, + "refId": "D" + } + ], + "title": "Autoscaling (Pods & Nodes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "If container xyz is hitting 100% of its CPU request, then it's a performance bottleneck. Increase its request (and possibly the size of the node it is on) and it should run faster.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0.3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "mappings": [], + "max": 200, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "green", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 7, + "y": 1 + }, + "id": 200, + "links": [], + "maxDataPoints": 10, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (irate(container_cpu_usage_seconds_total{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\", image!=\"\"}[4m])) \n/ \nsum by(pod,container) (kube_pod_container_resource_requests{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"cpu\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"}) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container CPU (% of request)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "mappings": [], + "max": 101, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "yellow", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 11, + "y": 1 + }, + "id": 94, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(100*(\n sum by(pod,container) (irate(container_cpu_usage_seconds_total {node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", job=\"kubelet\", container!=\"POD\", image!=\"\"}[4m])) \n/ sum by(pod,container) ( kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", job=\"kube-state-metrics\", resource=\"cpu\" } )\n))\n# {{container}}\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(100*(\n sum by(pod,container) (irate(container_cpu_usage_seconds_total {node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container=\"sas-programming-environment\", job=\"kubelet\", container!=\"POD\", image!=\"\"}[4m])) \n/ sum by(pod,container) ( kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container=\"sas-programming-environment\", job=\"kube-state-metrics\", resource=\"cpu\" } )\n))\n", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SPRE: {{pod}}", + "range": true, + "refId": "C" + } + ], + "title": "Container CPU (% of limit)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "max": 1.01, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 15, + "y": 1 + }, + "id": 95, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (nodename) (\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[$__rate_interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"})\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Node CPU Util", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 200000000, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 0.7, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#408040", + "value": 500000000 + }, + { + "color": "#1eff00", + "value": 1000000000 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/ Receive/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ Transmit/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 19, + "y": 1 + }, + "id": 219, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (nodename,device) (rate(node_network_receive_bytes_total{job=\"node-exporter\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", device!=\"lo\", device!~\"veth.*\", device!~\"en.*\"}[$__rate_interval]) )", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Rx: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (nodename,device) (rate(node_network_transmit_bytes_total{job=\"node-exporter\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", device!=\"lo\", device!~\"veth.*\", device!~\"en.*\"}[$__rate_interval]) )", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Tx: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Node Network Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 0.3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "mappings": [], + "max": 200, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "green", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 7, + "y": 8 + }, + "id": 100, + "links": [], + "maxDataPoints": 10, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_rss{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_requests{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\r\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory (rss) (% of request)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "If near 100% then processes in the container are at risk of OutOfMemory exception", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "mappings": [], + "max": 101, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#ff0000", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 11, + "y": 8 + }, + "id": 96, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_rss{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory (rss) (% of limit)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "rss + dirty pages (in cache but not safe to reclaim / flush from mem due to not yet being written to disk)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "max": 1.01, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#ff0000", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 15, + "y": 8 + }, + "id": 143, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# graph1: total-available ~= memrss ... ((total-available)+dirtypages) = memrss+dirtypages\n# graph2: total-free ~= memrss+cached+buffers ... ((total-free)+dirtypages) = memrss+dirtypages+cached+buffers\n#( (node_memory_MemTotal_bytes{nodename=~\"$Node\"}-node_memory_MemAvailable_bytes{nodename=~\"$Node\"}) + node_memory_Dirty_bytes{nodename=~\"$Node\"}) / node_memory_MemTotal_bytes{nodename=~\"$Node\"}\nsum by(nodename)(\n ( (node_memory_MemTotal_bytes{nodename=~\"$Node\"}-node_memory_MemAvailable_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}) + node_memory_Dirty_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}) / node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( instance:node_memory_utilisation:ratio{job=\"node-exporter\", nodename=~\"$Node\"}\n / ignoring (instance) group_left count without (instance) (instance:node_memory_utilisation:ratio{job=\"node-exporter\", nodename=~\"$Node\"})\n) + ( node_memory_Dirty_bytes{nodename=~\"$Node\"} / node_memory_MemAvailable_bytes{nodename=~\"$Node\"} )", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Node Memory Util (rss+dirty)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 200000000, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 0.7, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#408040", + "value": 1000000000 + }, + { + "color": "#1eff00", + "value": 5000000000 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 19, + "y": 8 + }, + "id": 151, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(node_disk_read_bytes_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[1m])) by (nodename,device)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Read: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(node_disk_written_bytes_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[1m])) by (nodename,device)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Write: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Node Disk Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "This panel is useful for seeing how long it takes SAS Viya pods to progress during startup to all pods reaching ContainerReady=true ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "viya containers total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#37872D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya containers ready" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya init containers total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1F60C4", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya init containers ready" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya containers waiting" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya init containers waiting" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C0D8FF", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 0, + "y": 9 + }, + "id": 49, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "count by (namespace) (kube_pod_container_status_ready{namespace=~\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} containers total", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum by (namespace) (kube_pod_container_status_ready{namespace=~\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 10, + "legendFormat": "{{namespace}} containers ready", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (namespace) (kube_pod_container_status_waiting{namespace=~\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} containers waiting", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "count by (namespace) (kube_pod_init_container_status_ready{namespace=~\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} init containers total", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (namespace) (kube_pod_init_container_status_ready{namespace=~\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} init containers ready", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (namespace) (kube_pod_init_container_status_waiting{namespace=~\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} init containers waiting", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": " sum by(namespace) (kube_pod_container_status_ready{namespace=~\"$namespace\"\r\n , container!=\"sas-opendistro-operator\"\r\n , container!=\"sas-readiness\"\r\n , container!=\"sas-prepull\"\r\n , container!=\"sas-data-server-utility\"\r\n , container!=\"sas-connect-spawner\"\r\n , container!~\"sas-crunchy-data-postgres.*\"\r\n , container!=\"sas-connect-spawner\"\r\n , container!=\"sas-config-reconciler\"\r\n , container!=\"sas-cas-operator\"\r\n , container!=\"prometheus-pushgateway\"\r\n , container!=\"sas-data-server-operator\"}) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "G" + } + ], + "title": "Container Status Count by Namespace", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Two metrics in one graph, both about \"storage use\":\n\n1. node local storage like ext4, xfs\nexcluding tmpfs & fuse.lxcfs\nAlso, switching from filesystem size-available to size-free because this is what df outputs (since freebytes = availbytes + rootreserved, and rootreserved is not really available to users).\n\n2. PVC usage. Warning: PVCs provisioned from Azure NFS server show total NFS server \"/export\" LUN capacity and usage percentages for each PVC, instead of the actual size and usage of each distinct PVC... Not sure (yet) how to get the actual accurate PVC usage metrics.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "#ff0000", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 3, + "y": 9 + }, + "id": 144, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( sum without (device) (\n max without () (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"} \n - node_filesystem_free_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"}\n )\n ) \n/ ignoring (instance) group_left\n sum without (instance, device) (\n max without () (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"}\n )\n )\n) \n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "fs({{fstype}}) - {{nodename}}:{{mountpoint}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# PVC: {{persistentvolumeclaim}}\r\n# pvc({{persistentvolumeclaim}}) - ns({{namespace}})\r\n( kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"} \r\n- kubelet_volume_stats_available_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"})\r\n/ kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pvc({{persistentvolumeclaim}}) - ns({{namespace}})", + "range": true, + "refId": "A" + } + ], + "title": "Node Disk & PVC Storage Use %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "mappings": [], + "max": 200, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 7, + "y": 15 + }, + "id": 243, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_usage_bytes{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_requests{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory (tot) (% of request)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "If near 100% then processes in the container are at risk of OutOfMemory exception", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "mappings": [], + "max": 101, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 11, + "y": 15 + }, + "id": 242, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_usage_bytes{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory (tot) (% of limit)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "memory use (including filecache) which can easily reach 100% due to natural caching of any files opened in order to speed up any subsequent re-open.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "max": 1.01, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 15, + "y": 15 + }, + "id": 202, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# available - free = cached... fixing by changing this \"total mem use (rss+cache)\" to just MemAvailable\r\n# graph1: total-available ~= memrss ... ((total-available)+dirtypages) = memrss+dirtypages\r\n# graph2: total-free ~= memrss+cached+buffers\r\n# ( (node_memory_MemTotal_bytes{nodename=~\"$Node\"}-node_memory_MemFree_bytes{nodename=~\"$Node\"}) ) / node_memory_MemTotal_bytes{nodename=~\"$Node\"}\r\nsum by(nodename)(\r\n ( (node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}-node_memory_MemFree_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}) ) / node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}\r\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "D", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( node_memory_MemAvailable_bytes{nodename=~\"$Node\"} \r\n - node_memory_MemFree_bytes{nodename=~\"$Node\"} \r\n - node_memory_Cached_bytes{nodename=~\"$Node\"}\r\n - node_memory_Buffers_bytes{nodename=~\"$Node\"}\r\n #- node_memory_Dirty_bytes{nodename=~\"$Node\"}\r\n #? node_memory_Active_bytes{nodename=~\"$Node\"}\r\n) / node_memory_MemTotal_bytes{nodename=~\"$Node\"}\r\n# (available-free)/total = 2.600% to 67.90% ...\r\n# (available-free-cached)/total = -0.913% to 3.37% ... => available ~= free +cached (this is the % of memory \"available\" for use by processes )\r\n# (available-free-cached-buffers)/total = -0.977% to 2.23% ... => available ~= free +cached +buffers I think (total-free) = memRss what is node_memory_Active_bytes ?\r\n# (available-free-cached-dirty)/total = -2.500% to 3.36% ... (dirty pages are part of cache that are not yet on disk, so I think this line doesnt make sense)\r\n# graph1: total-available ~= memrss ... ((total-available)+dirtypages) = memrss+dirtypages\r\n# graph2: total-free ~= memrss+cached ... ((total-free)+dirtypages) = memrss+dirtypages+cached+buffers+etc\r\n", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}} totalMemUse", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Node Memory Util (tot) ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0.5, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 19, + "y": 15 + }, + "id": 214, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "#{{cas_server}}, {{pod}}, {{container}}, {{service}}, {{endpoint}}, {{instance}}\r\n(cas_grid_sessions_current{namespace=~\"$namespace\",service=~\".+-client\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "CAS Sessions", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 46, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 52, + "links": [], + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (nodename) (\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[$__rate_interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"})\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Node CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "rss + dirty only, filecache not included. dirty is kind of like lazy write bytes that not yet been written to disk so its not safe to flush them out of filecache yet.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 54, + "links": [], + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# graph1: total-available ~= memrss ... ((total-available)+dirtypages) = memrss+dirtypages\n# graph2: total-free ~= memrss+cached+buffers ... ((total-free)+dirtypages) = memrss+dirtypages+cached+buffers\n( (node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}-node_memory_MemAvailable_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}) + node_memory_Dirty_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"})\n/ node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Node Memory Util (rss+dirty)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 98, + "links": [], + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# available - free = cached... fixing by changing this \"total mem use (rss+cache)\" to just MemAvailable\r\n# graph1: total-available ~= memrss ... ((total-available)+dirtypages) = memrss+dirtypages\r\n# graph2: total-free ~= memrss+cached+buffers\r\n( (node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}-node_memory_MemFree_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}) )\r\n/ node_memory_MemTotal_bytes{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}} totalMemUse", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "D", + "step": 10 + } + ], + "title": "Node Memory Util (totalMemUse%, rss+cache+buffers)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Node CPU & Memory", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 196, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "user+nice cpu time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 3 + }, + "id": 157, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 500 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( sum by(nodename) ( irate( node_cpu_seconds_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\",mode=\"user\"}[1m] ) ) \r\n + sum by(nodename) ( irate( node_cpu_seconds_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", mode=\"nice\"}[1m] ) )\r\n) / sum(irate(node_cpu_seconds_total[1m])) by(nodename)\r\n", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "range": true, + "refId": "B" + } + ], + "title": "Node CPU Utilization (user+nice)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "system+irq+softirq", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 9 + }, + "id": 158, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 500 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( sum by(nodename) ( irate( node_cpu_seconds_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\",mode=\"system\"}[1m] ) ) \r\n + sum by(nodename) ( irate( node_cpu_seconds_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", mode=\"irq\"}[1m] ) )\r\n + sum by(nodename) ( irate( node_cpu_seconds_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", mode=\"softirq\"}[1m] ) )\r\n) / sum(irate(node_cpu_seconds_total[1m])) by(nodename)\r\n", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "range": true, + "refId": "B" + } + ], + "title": "Node CPU Utilization (system)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 15 + }, + "id": 159, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 500 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( sum by(nodename) ( irate( node_cpu_seconds_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\",mode=\"iowait\"}[1m] ) ) \r\n) / sum(irate(node_cpu_seconds_total[1m])) by(nodename)\r\n", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{nodename}}", + "range": true, + "refId": "B" + } + ], + "title": "Node CPU Utilization (iowait)", + "type": "timeseries" + } + ], + "title": "Node CPU (detail)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 61, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 3, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 3, + "links": [], + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(pod,container) (irate(container_cpu_usage_seconds_total{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\", image!=\"\"}[4m]))\r\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ container }}", + "range": true, + "refId": "A" + } + ], + "title": "Container CPU Use", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 99, + "links": [], + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(pod,container) (container_memory_rss{node=~\"$Node\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", pod!~\".*prepull.*\", container=~\"$container\", container!=\"\", container!=\"POD\", container!=\"\"})\r\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{container}}", + "range": true, + "refId": "D" + } + ], + "title": "Container Memory Use (rss)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 217, + "links": [], + "maxDataPoints": 10000, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(pod,container) (container_memory_usage_bytes{node=~\"$Node\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", pod!~\".*prepull.*\", container=~\"$container\", container!=\"\", container!=\"POD\", container!=\"\"})\r\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{container}}", + "range": true, + "refId": "D" + } + ], + "title": "Container Memory Use (tot)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Container Utilization", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 149, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "max": 200, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 116, 217, 0.2)" + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 5 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 700 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (irate(container_cpu_usage_seconds_total{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\", image!=\"\"}[4m])) \n/ \nsum by(pod,container) (kube_pod_container_resource_requests{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"cpu\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"}) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container CPU use as a % of request", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "max": 200, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 116, 217, 0.2)" + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 11 + }, + "id": 201, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 700 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_rss{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_requests{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory use (rss) as a % of request", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "max": 200, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 116, 217, 0.2)" + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 17 + }, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 700 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_usage_bytes{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_requests{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\r\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory use (tot) as a % of request", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Container Utilization (% of Request)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 147, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#25dc0c" + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 6 + }, + "id": 57, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 700 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (irate(container_cpu_usage_seconds_total{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\", image!=\"\"}[4m])) \n/ \nsum by(pod,container) (kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"cpu\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"}) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}/{{container}}", + "refId": "A" + } + ], + "title": "Container CPU use as a % of limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "If near 100% then processes in the container are at risk of OutOfMemory exception or oom killer", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 12 + }, + "id": 101, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 700 + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_rss{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}/{{container}}", + "refId": "A" + } + ], + "title": "Container Memory use (rss) as a % of limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "If near 100% then processes in the container are at risk of OutOfMemory exception", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 100 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 23, + "x": 0, + "y": 18 + }, + "id": 59, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 700 + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "( 100 * (\nsum by(pod,container) (container_memory_usage_bytes{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kubelet\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}) \n/ \nsum by(pod,container) (kube_pod_container_resource_limits{node=~\"$Node\", container!=\"\", pod!~\".*prepull.*\", job=\"kube-state-metrics\", resource=\"memory\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\" }) \n))\n", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}/{{container}}", + "refId": "A" + } + ], + "title": "Container Memory use (tot) as a % of limit", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Container Utilization (% of Limit)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 137, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/ Receive/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ Transmit/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 7 + }, + "id": 127, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (nodename,device) (rate(node_network_receive_bytes_total{job=\"node-exporter\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", device!=\"lo\", device!~\"veth.*\"}[$__rate_interval]) )", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Receive: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (nodename,device) (rate(node_network_transmit_bytes_total{job=\"node-exporter\", nodename=~\"$Node\", nodename=~\".*$NodeClass.*\", device!=\"lo\", device!~\"veth.*\"}[$__rate_interval]) )", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Transmit: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Net Utilization (by node/interface)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 11, + "y": 7 + }, + "id": 106, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (node,device) (irate(container_network_receive_packets_total{node=~\"$Node\", node=~\".*$NodeClass.*\", namespace=\"\", interface!=\"lo\", interface!~\"veth.*\"}[$__rate_interval]) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Rx-Node: {{node}} / {{interface}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(irate(container_network_transmit_packets_total{node=~\"$Node\", node=~\".*$NodeClass.*\", namespace=\"\", interface!=\"lo\", interface!~\"veth.*\"}[$__rate_interval]) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Tx-Node: {{node}} / {{interface}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "refId": "D", + "step": 10 + } + ], + "title": "Network packets/second (per node) - inbound (Rx) & outbound (Tx)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Node Network", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 130, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 8 + }, + "id": 131, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(node_disk_read_bytes_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[1m])) by (nodename,device)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Read: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(node_disk_written_bytes_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[1m])) by (nodename,device)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Write: {{nodename}} ({{device}})", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Node Disk Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 11, + "y": 8 + }, + "id": 132, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(node_disk_reads_completed_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[1m]) ", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Read: {{nodename}} ({{device}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(node_disk_writes_completed_total{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}[1m]) ", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Write: {{nodename}} ({{device}})", + "range": true, + "refId": "B" + } + ], + "title": "Node Disk IOPS", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Node Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 161, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 9 + }, + "id": 163, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_sockstat_sockets_used{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{nodename}} (used)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_sockstat_TCP_alloc{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{nodename}} (TCP_alloc)", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_sockstat_TCP_inuse{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{nodename}} (TCP_inuse)", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_sockstat_TCP_tw{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{nodename}} (TCP_tw)", + "range": true, + "refId": "D" + } + ], + "title": "node_sockstat_*", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 11, + "y": 9 + }, + "id": 164, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_netstat_Tcp_CurrEstab{nodename=~\"$Node\", nodename=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{nodename}}", + "range": true, + "refId": "A" + } + ], + "title": "node_netstat_Tcp_CurrEstab", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Node Sockets", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 135, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Node local storage (e.g. ext4, xfs, etc.) excluding tmpfs & fuse.lxcfs\n\nUses (filesystem size-free) to be comparable to Linux df outputs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 22 + }, + "id": 227, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( sum without (device) (\n max without () (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"} \n - node_filesystem_free_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"}\n )\n ) \n/ ignoring (instance) group_left\n sum without (instance, device) (\n max without () (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"}\n )\n )\n) \n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "fs({{fstype}}) - {{nodename}}:{{mountpoint}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "C", + "step": 10 + } + ], + "title": "Disk Capacity Used %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Warning: PVCs provisioned from Azure NFS server show total NFS server \"/export\" LUN capacity and usage percentages for each PVC, instead of the actual size and usage of each distinct PVC", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 11, + "y": 22 + }, + "id": 228, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# PVC: {{persistentvolumeclaim}}\r\n# pvc({{persistentvolumeclaim}}) - ns({{namespace}})\r\n( kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"} \r\n- kubelet_volume_stats_available_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"})\r\n/ kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pvc({{persistentvolumeclaim}}) - ns({{namespace}})", + "range": true, + "refId": "A" + } + ], + "title": "PVC Usage %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Node local storage (e.g. ext4, xfs, etc.) excluding tmpfs & fuse.lxcfs\n\nUses (filesystem size-free) to be comparable to Linux df outputs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 11, + "x": 0, + "y": 34 + }, + "id": 229, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "( sum without (device) (\n max without () (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"} \n - node_filesystem_free_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"}\n )\n ) \n#/ ignoring (instance) group_left\n# sum without (instance, device) (\n# max without () (\n# node_filesystem_size_bytes{job=\"node-exporter\", instance=~\"$instance\", fstype!=\"tmpfs\", fstype!=\"fuse.lxcfs\", nodename=~\".*$NodeClass.*\"}\n# )\n# )\n) \n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "fs({{fstype}}) - {{nodename}}:{{mountpoint}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "range": true, + "refId": "C", + "step": 10 + } + ], + "title": "Disk Capacity Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Warning: PVCs provisioned from Azure NFS server show total NFS server \"/export\" LUN capacity and usage percentages for each PVC, instead of the actual size and usage of each distinct PVC", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "transparent", + "value": 1 + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 11, + "y": 34 + }, + "id": 230, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# PVC: {{persistentvolumeclaim}}\r\n# pvc({{persistentvolumeclaim}}) - ns({{namespace}})\r\n( kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"} \r\n- kubelet_volume_stats_available_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"})\r\n#/ kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"$pvc\", namespace=~\"$namespace\", node=~\".*$NodeClass.*\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pvc({{persistentvolumeclaim}}) - ns({{namespace}})", + "range": true, + "refId": "A" + } + ], + "title": "PVC Usage", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Storage Use", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 142, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pg_settings_max_connections" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "db connections: ns=viya: golang" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "db connections: ns=viya: spring" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 23 + }, + "id": 68, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": " ( sum by (namespace) (sas_db_pool_connections{container=~\"$container\", pod=~\"$pod\", node=~\"$Node\"}) )\n+ ( sum by (namespace) (jdbc_connections_active{container=~\"$container\", pod=~\"$pod\", node=~\"$Node\"}) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "db connections: ns={{namespace}}: TOTAL (go+java)", + "refId": "D" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "pg_settings_max_connections", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pg_settings_max_connections", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": " ( sum by (namespace,sas_service_base) (sas_db_pool_connections{container=~\"$container\", pod=~\"$pod\", node=~\"$Node\"}) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "db connections: ns={{namespace}}: {{sas_service_base}}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": " ( sum by (namespace,sas_service_base) (jdbc_connections_active{container=~\"$container\", pod=~\"$pod\", node=~\"$Node\"}) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "db connections: ns={{namespace}}: {{sas_service_base}}", + "refId": "C" + } + ], + "title": "db connections active (total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 11, + "y": 23 + }, + "id": 69, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (namespace,container,sas_service_base) (sas_db_pool_connections{container=~\"$container\", pod=~\"$pod\", node=~\"$Node\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}: {{container}} ({{sas_service_base}})", + "refId": "D" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (namespace,container,sas_service_base) (jdbc_connections_active{container=~\"$container\", pod=~\"$pod\", node=~\"$Node\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}: {{container}} ({{sas_service_base}})", + "refId": "A" + } + ], + "title": "db connections active (by container)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Postgres", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 195, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "if > 80% then scaleUp 1 node ... really if any pod cant find a node to schedule to, this is what triggers scale out of a new node\n\nif < 50% then scaleDown 1 node (for avg req allocation mem & cpu on any given node)\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + }, + { + "color": "transparent", + "value": 0.5 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "aks-generic: memory", + "aks-system: memory", + "aks-cas: memory" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 191, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (nodeprefix, resource)(\r\n label_replace ( \r\n max by(pod, node, resource)( \r\n kube_pod_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\"} or\r\n kube_pod_init_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\"}\r\n )\r\n ,\"nodeprefix\", \"$1\", \"node\", \"(aks-[^-]+).+\")\r\n)\r\n/ sum by (nodeprefix, resource)(\r\n label_replace ( \r\n kube_node_status_allocatable {node=~\"$Node\", node=~\".*$NodeClass.*\"}\r\n ,\"nodeprefix\", \"$1\", \"node\", \"(aks-[^-]+).+\"\r\n )\r\n )", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{nodeprefix}}: {{resource}}", + "range": true, + "refId": "B" + } + ], + "title": "ClusterAutoscaler Trigger: Average(%Request Allocated) Per Nodepool ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "ClusterAutoscaler ScaleUp happens when at least one pod is unable to be scheduled.\n\nClusterAutoscaler ScaleDown happens when node's cpu AND memory requests/capacity is below the scale-down-utilization-threshold of 50%(default), denoted by the lower blue region\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + }, + { + "color": "transparent", + "value": 0.5 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 8, + "y": 24 + }, + "id": 190, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# This query reflects what you see from \"kubectl describe node | grep 'Allocated resources' -A6\r\n# Divide by \"allocatable\" cpu or memory instead of \"capacity\"\r\n# Use the highest value for cpu request & memory req from all containers And initContainers in each pod\r\nsum by(node)(\r\n max by(pod, node, resource)( \r\n kube_pod_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"cpu\"} or\r\n kube_pod_init_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"cpu\"} )\r\n) / sum by(node)( kube_node_status_allocatable {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"cpu\"} )", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "ClusterAutoscaler Trigger: CPU %Request Allocated Per Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "ClusterAutoscaler ScaleUp happens when at least one pod is unable to be scheduled.\n\nClusterAutoscaler ScaleDown happens when node's cpu AND memory requests/capacity is below the scale-down-utilization-threshold of 50%(default), denoted by the lower blue region\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + }, + { + "color": "transparent", + "value": 0.5 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 15, + "y": 24 + }, + "id": 193, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": false + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# This query reflects what you see from \"kubectl describe node | grep 'Allocated resources' -A6\r\n# Divide by \"allocatable\" cpu or memory instead of \"capacity\"\r\n# Use the highest value for cpu request & memory req from all containers And initContainers in each pod\r\nsum by(node)(\r\n max by(pod, node, resource)( \r\n kube_pod_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"memory\"} or\r\n kube_pod_init_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"memory\"} )\r\n) / sum by(node)( kube_node_status_allocatable {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"memory\"} )\r\n", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{node}}: mem", + "range": true, + "refId": "B" + } + ], + "title": "ClusterAutoscaler Trigger: Mem %Request Allocated Per Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "ClusterAutoscaler ScaleDown should happen for any node in this table with \"blue\" CPU & MEM (i.e. below the scale-down-utilization-threshold of 50%(default)). For more detail -> https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#how-does-scale-down-work\n\nIf nodes are not scaling down, then check nodes for Pods with restrictive PodDisruptionBudget, Pods that are not backed by a controller object, etc. For more detail -> https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-types-of-pods-can-prevent-ca-from-removing-a-node\n\n\nClusterAutoscaler ScaleUp happens when at least one pod is unable to be scheduled. Numbers over 80% are colored red to indicate increased chance of not being able to schedule a pod on that node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-text" + }, + "filterable": true, + "inspect": true + }, + "decimals": 1, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + }, + { + "color": "text", + "value": 0.5 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cpu" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mem" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "eph" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 194, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# This query reflects what you see from \"kubectl describe node | grep 'Allocated resources' -A6\r\n# Divide by \"allocatable\" cpu or memory instead of \"capacity\"\r\n# Use the highest value for cpu request & memory req from all containers And initContainers in each pod\r\nsum by(node)(\r\n max by(pod, node, resource)( \r\n kube_pod_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"cpu\"} or\r\n kube_pod_init_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"cpu\"} )\r\n) / sum by(node)( kube_node_status_allocatable {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"cpu\"} )", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{node}}: cpu", + "range": false, + "refId": "cpu" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# This query reflects what you see from \"kubectl describe node | grep 'Allocated resources' -A6\r\n# Divide by \"allocatable\" cpu or memory instead of \"capacity\"\r\n# Use the highest value for cpu request & memory req from all containers And initContainers in each pod\r\nsum by(node)(\r\n max by(pod, node, resource)( \r\n kube_pod_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"memory\"} or\r\n kube_pod_init_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"memory\"} )\r\n) / sum by(node)( kube_node_status_allocatable {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"memory\"} )\r\n", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{node}}: mem", + "range": false, + "refId": "memory" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# This query reflects what you see from \"kubectl describe node | grep 'Allocated resources' -A6\r\n# Divide by \"allocatable\" cpu or memory instead of \"capacity\"\r\n# Use the highest value for cpu request & memory req from all containers And initContainers in each pod\r\nsum by(node)(\r\n max by(pod, node, resource)( \r\n kube_pod_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"ephemeral_storage\"} or\r\n kube_pod_init_container_resource_requests {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"ephemeral_storage\"} )\r\n) / sum by(node)( kube_node_status_allocatable {node=~\"$Node\", node=~\".*$NodeClass.*\", resource=\"ephemeral_storage\"} )", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{node}}: ephemeral_storage", + "range": false, + "refId": "ephemeral_storage" + } + ], + "title": "ClusterAutoscaler Trigger: %Request Allocated Per Node", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "node", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true + }, + "indexByName": {}, + "renameByName": { + "Value #cpu": "cpu", + "Value #ephemeral_storage": "eph", + "Value #memory": "mem" + } + } + } + ], + "type": "table" + } + ], + "title": "Cluster Autoscaler (Azure AKS Only)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 139, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Watch the sas-readiness container to see when viya begins starting up (container exists but ready=0), and viya startup complete (ready=1)\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 5, + "x": 0, + "y": 25 + }, + "id": 205, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (container,pod) (kube_pod_container_status_ready{namespace=~\"$namespace\", container=\"sas-readiness\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "VIYA_UP({{container}})", + "refId": "A" + } + ], + "title": "Is Viya Up?", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "container restarts\n(Not a \"pod Terminated/restart\" count)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1eff00" + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 5, + "y": 25 + }, + "id": 206, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "increase(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"}[1m])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "restarts({{container}})", + "refId": "B" + } + ], + "title": "Container Restarts", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Health: pods & containers", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 168, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 10, + "x": 0, + "y": 26 + }, + "id": 166, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "# ({{service}}) > ({{endpoint}}) > ({{sas_service_base}}) > ({{node}})\r\nprocess_open_fds{namespace=~\"$namespace\", pod=~\"$pod\", service!=\"\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "({{service}}) > ({{endpoint}}) > ({{sas_service_base}}) > ({{node}})", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "process_open_fds{namespace=~\"$namespace\", pod=~\"$pod\", service=\"\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "B" + } + ], + "title": "Open File Descriptors per Pod -- (endpoint) > (sas_service_base) > (service) > (node)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "File Descriptors", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 65, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "viya: HPA pod replicas (desired)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya: HPA pod replicas (ready)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 0, + "y": 27 + }, + "id": 76, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "count( container_start_time_seconds{node=~\"$Node\",namespace=~\"$namespace\",container=~\"sas-programming-environment\"} )", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "SAS Programming container count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "http requests seen from the perspective of the nginx-ingress \"front door\". 1 minute moving window, so each data point shows the hits/second averaged over the past 1 minute.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 7, + "y": 27 + }, + "id": 42, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "# sum by(exported_namespace) ( irate(nginx_ingress_controller_requests{job=\"ingress-nginx\", exported_namespace=~\"$namespace\", service=~\"$container\"}[1m]) ) \n#clamp_max( sum by(exported_namespace) ( increase(nginx_ingress_controller_requests{job=\"ingress-nginx\", exported_namespace=~\"$namespace\", service=~\"$container\"}[1m]) ), 10000 )\n sum by(exported_namespace) ( irate(nginx_ingress_controller_requests{job=\"ingress-nginx\", exported_namespace=~\"$namespace\", service=~\"$container\"}[1m]) )", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{exported_namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "nginx_ingress_controller_requests (hits/second) by namespace", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "time(podContainersReady) - time(podScheduled)\n\napproximate \"group by deployment|statefulset\" by using label_replace function and handling 3 different cases\n1. two random hash strings (or seq numbers) after 2nd to last dash\n2. one sequence number after last dash\n3. no random hash or sequence numbers\n\nif multiple pod replicas in a deployment or statefulset, display max value\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 14, + "y": 27 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# debugging: list all without any label_replace grouping by \"app base name\"\r\nmax by(pod) (\r\n kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*\"} \r\n - kube_pod_status_scheduled_time{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*\"}\r\n)", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# primary case: two random hashes after second to last dash at end of \"base app name\"\r\nmax by(podprefix) (\r\n label_replace( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*|sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+-[^-]+$\" )\r\n - label_replace( kube_pod_status_scheduled_time{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*|sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+-[^-]+$\" )\r\n)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{podprefix}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# special case: only sequence number at end of \"base app name\"\r\nmax by(podprefix) (\r\n label_replace( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-default-worker.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+$\" )\r\n - label_replace( kube_pod_status_scheduled_time{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-default-worker.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+$\" )\r\n)\r\n", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{podprefix}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# special case: no random hashes or sequence numbers at end of \"base app name\"\r\nmax by(podprefix) (\r\n label_replace( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-cas-server-.*-controller\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)$\" )\r\n - label_replace( kube_pod_status_scheduled_time{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-cas-server-.*-controller\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)$\" )\r\n)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{podprefix}}", + "range": true, + "refId": "D" + } + ], + "title": "Pod Startup Time (wait+init+ready)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "viya: HPA pod replicas (desired)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "viya: HPA pod replicas (ready)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 0, + "y": 36 + }, + "id": 77, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "count by(node)( container_start_time_seconds{node=~\"$Node\",namespace=~\"$namespace\",container=\"sas-programming-environment\"} )\r\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + } + ], + "title": "SAS Programming containers per node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "http requests seen from the perspective of the nginx-ingress \"front door\". 1 minute moving window, so each data point shows the hits/second averaged over the past 1 minute.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 7, + "y": 36 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "#sum by (service,status) ( irate(nginx_ingress_controller_requests{job=\"ingress-nginx\", exported_namespace=~\"$namespace\", service=~\"$container\"}[1m]) )\n#sum by (service,path) ( increase(nginx_ingress_controller_requests{job=\"ingress-nginx\", exported_namespace=~\"$namespace\", service=~\"$container\"}[1m]) )\n sum by (service,path) ( irate(nginx_ingress_controller_requests{job=\"ingress-nginx\", exported_namespace=~\"$namespace\", service=~\"$container\"}[1m]) )\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{service}}:{{path}}", + "range": true, + "refId": "C" + } + ], + "title": "nginx_ingress_controller_requests (hits/second) by service:endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "time(podContainersReady) - time(podScheduled)\n\napproximate \"group by deployment|statefulset\" by using label_replace function and handling 3 different cases\n1. two random hash strings (or seq numbers) after 2nd to last dash\n2. one sequence number after last dash\n3. no random hash or sequence numbers\n\nif multiple pod replicas in a deployment or statefulset, display max value\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 14, + "y": 36 + }, + "id": 192, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# debug\r\n max by(pod) ( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*\"} )\r\n- max by(pod) ( kube_pod_container_state_started{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*\"} )", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# primary case: two random hashes after second to last dash at end of \"base app name\"\r\n max by(podprefix) (label_replace( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*|sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+-[^-]+$\" ) )\r\n- max by(podprefix) (label_replace( kube_pod_container_state_started{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$container.*\", pod!~\"sas-readiness.*|sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+-[^-]+$\" ) )", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{podprefix}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# special case: only sequence number at end of \"base app name\"\r\n max by(podprefix) ( label_replace( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-default-worker.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+$\" ) )\r\n- max by(podprefix) ( label_replace( kube_pod_container_state_started{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-consul-server.*|sas-rabbitmq-server.*|sas-opendistro-default.*|sas-workload-orchestrator.*|sas-cas-server-default-worker.*\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)-[^-]+$\" ) )\r\n", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{podprefix}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# special case: no random hashes or sequence numbers at end of \"base app name\"\r\n max by(podprefix) ( label_replace( kube_pod_status_ready_time {namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-cas-server-.*-controller\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)$\" ) )\r\n- max by(podprefix) ( label_replace( kube_pod_container_state_started{namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"sas-cas-server-.*-controller\"}, \"podprefix\", \"$1\", \"pod\", \"^(.+)$\" ) )", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{podprefix}}", + "range": true, + "refId": "D" + } + ], + "title": "Pod Startup Time (ready)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Workload", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 63, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "\"desired\" number of pod replicas from the perspective of the HorizontalPodAutoscaler", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 28 + }, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "#sum by(hpa) (kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"$namespace\"})\r\nsum by(horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"$namespace\"})\r\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{horizontalpodautoscaler}}", + "refId": "A" + } + ], + "title": "Pod Replicas (by HPA)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of pod replicas that have reached \"ContainersReady\" status (ie. containers in those pods have readiness probe \"ready=true\" and are ready to process HTTP requests).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 11, + "y": 28 + }, + "id": 35, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(deployment) (kube_deployment_status_replicas_available{namespace=~\"$namespace\", deployment=~\"$pod\"})\r\n", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{deployment}}", + "refId": "A" + } + ], + "title": "Replicas (by Deployment)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Pod replicas \"scheduled\" lags behind \"desired\" due to node startup time (this is normal). \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "HPA pod replicas desired (viya)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HPA pod replicas ready (viya)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0B400", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 37 + }, + "id": 203, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(namespace) (kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"$namespace\"})\r\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "HPA pod replicas desired ({{namespace}})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(namespace) (kube_deployment_status_replicas_available{namespace=~\"$namespace\", deployment=~\"$pod\"\r\n , deployment!=\"sas-opendistro-operator\"\r\n , deployment!=\"sas-readiness\"\r\n , deployment!=\"sas-prepull\"\r\n , deployment!=\"sas-data-server-utility\"\r\n , deployment!=\"sas-connect-spawner\"\r\n , deployment!~\"sas-crunchy-data-postgres.*\"\r\n , deployment!=\"sas-connect-spawner\"\r\n , deployment!=\"sas-config-reconciler\"\r\n , deployment!=\"sas-cas-operator\"\r\n , deployment!=\"prometheus-pushgateway\"\r\n , deployment!=\"sas-data-server-operator\"\r\n})", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "HPA pod replicas ready ({{namespace}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(sum by (label_agentpool) (kube_node_labels{node=~\"$Node\"}))\r\n", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "nodepool: {{label_agentpool}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (status) (kube_node_status_condition{condition=\"Ready\", status=\"true\"})", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total \"Ready\" Nodes", + "range": true, + "refId": "D" + } + ], + "title": "Autoscaling (Pods & Nodes)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Autoscaling Analysis", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "kubernetes-cluster", + "welcome-cluster" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "xxxPBFA97CFB590B2093" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_node_labels,label_workload_sas_com_class)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "NodeClass", + "options": [], + "query": { + "query": "label_values(kube_node_labels,label_workload_sas_com_class)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_node_labels{label_workload_sas_com_class=~\"$NodeClass\"},node)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Node", + "options": [], + "query": { + "query": "label_values(kube_node_labels{label_workload_sas_com_class=~\"$NodeClass\"},node)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_uname_info{exported_nodename=~\"(?i:($Node))\"}, instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(node_uname_info{exported_nodename=~\"(?i:($Node))\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "pod", + "options": [], + "query": { + "query": "label_values(kube_pod_info{namespace=~\"$namespace\"}, pod)", + "refId": "Prometheus-pod-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(kube_pod_container_info{namespace=~\"$namespace\", pod=~\"$pod\"}, container)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "container", + "options": [], + "query": { + "query": "label_values(kube_pod_container_info{namespace=~\"$namespace\", pod=~\"$pod\"}, container)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}, persistentvolumeclaim)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "pvc", + "options": [], + "query": { + "query": "label_values(kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}, persistentvolumeclaim)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Perf / Analysis", + "uid": "c36daf03-6384-4b2a-b2e2-e2c0e205822d3", + "version": 1, + "weekStart": "" +} \ No newline at end of file