diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index c06107d2669..394e11d2070 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -136,15 +136,23 @@ helm uninstall cl-cluster ``` # Grafana dashboard -We are using [Grabana]() lib to create dashboards programmatically +We are using [Grabana](https://github.com/K-Phoen/grabana) lib to create dashboards programmatically + +You can select `PANELS_INCLUDED`, options are `core`, `wasp`, comma separated + +You can also select dashboard platform in `INFRA_PLATFORM` either `kubernetes` or `docker` ``` +export LOKI_TENANT_ID=promtail +export LOKI_URL=... export GRAFANA_URL=... export GRAFANA_TOKEN=... -export LOKI_DATA_SOURCE_NAME=Loki export PROMETHEUS_DATA_SOURCE_NAME=Thanos -export DASHBOARD_FOLDER=CRIB -export DASHBOARD_NAME=ChainlinkCluster +export LOKI_DATA_SOURCE_NAME=Loki +export PANELS_INCLUDED=core,wasp +export INFRA_PLATFORM=kubernetes|docker +export GRAFANA_FOLDER=DashboardCoreDebug +export DASHBOARD_NAME=ChainlinkClusterDebug -cd dashboard/cmd && go run dashboard_deploy.go +go run dashboard/cmd/dashboard_deploy.go ``` -Open Grafana folder `CRIB` and find dashboard `ChainlinkCluster` \ No newline at end of file +Open Grafana folder `DashboardCoreDebug` and find dashboard `ChainlinkClusterDebug` \ No newline at end of file diff --git a/charts/chainlink-cluster/dashboard/dashboard.go b/charts/chainlink-cluster/dashboard/dashboard.go index bc5fa51cc31..ff8b7fa8a0a 100644 --- a/charts/chainlink-cluster/dashboard/dashboard.go +++ b/charts/chainlink-cluster/dashboard/dashboard.go @@ -12,7 +12,10 @@ import ( ) type PanelOption struct { - labelFilter string + labelFilters map[string]string + labelFilter string + legendString string + labelQuery string } type Dashboard struct { @@ -71,8 +74,8 @@ func NewDashboard( "branch": `=~"${branch:pipe}"`, "commit": `=~"${commit:pipe}"`, } - waspPanelsLoadStats := wasp.WASPLoadStatsRow(db.PrometheusDataSourceName, panelQuery) - waspPanelsDebugData := wasp.WASPDebugDataRow(db.PrometheusDataSourceName, panelQuery, false) + waspPanelsLoadStats := wasp.WASPLoadStatsRow(db.LokiDataSourceName, panelQuery) + waspPanelsDebugData := wasp.WASPDebugDataRow(db.LokiDataSourceName, panelQuery, false) db.opts = append(db.opts, waspPanelsLoadStats) db.opts = append(db.opts, waspPanelsDebugData) break @@ -119,15 +122,33 @@ func (m *Dashboard) init() { dashboard.Tags(m.grafanaTags), } + m.panelOption.labelFilters = map[string]string{ + "instance": `=~"${instance}"`, + "commit": `=~"${commit:pipe}"`, + } + switch m.platform { case "kubernetes": + m.panelOption.labelFilters = map[string]string{ + "job": `=~"${instance}"`, + "namespace": `=~"${namespace}"`, + "pod": `=~"${pod}"`, + } m.panelOption.labelFilter = "job" + m.panelOption.legendString = "pod" break case "docker": + m.panelOption.labelFilters = map[string]string{ + "instance": `=~"${instance}"`, + } m.panelOption.labelFilter = "instance" + m.panelOption.legendString = "instance" break } + for key, value := range m.panelOption.labelFilters { + m.panelOption.labelQuery += key + value + ", " + } m.opts = append(m.opts, opts...) } @@ -158,15 +179,23 @@ func (m *Dashboard) addKubernetesVariables() { opts := []dashboard.Option{ dashboard.VariableAsQuery( "namespace", - query.DataSource(m.LokiDataSourceName), + query.DataSource(m.PrometheusDataSourceName), query.Multiple(), query.IncludeAll(), query.Request(fmt.Sprintf("label_values(%s)", "namespace")), query.Sort(query.NumericalAsc), ), + dashboard.VariableAsQuery( + "pod", + query.DataSource(m.PrometheusDataSourceName), + query.Multiple(), + query.IncludeAll(), + query.Request("label_values(kube_pod_container_info{namespace=\"$namespace\"}, pod)"), + query.Sort(query.NumericalAsc), + ), } m.opts = append(m.opts, opts...) - waspVariables := wasp.AddVariables(m.PrometheusDataSourceName) + waspVariables := wasp.AddVariables(m.LokiDataSourceName) m.opts = append(m.opts, waspVariables...) } diff --git a/charts/chainlink-cluster/dashboard/panels.go b/charts/chainlink-cluster/dashboard/panels.go index 597c4faf679..61161a2f7b6 100644 --- a/charts/chainlink-cluster/dashboard/panels.go +++ b/charts/chainlink-cluster/dashboard/panels.go @@ -12,28 +12,21 @@ import ( ) func (m *Dashboard) addMainPanels() { - var balancePanelSpanSize float32 = 4 var panelsIncluded []row.Option var goVersionLegend string = "version" - if m.platform == "kubernetes" { - balancePanelSpanSize = 3 - goVersionLegend = "exported_version" - } - globalInfoPanels := []row.Option{ row.WithStat( "App Version", stat.DataSource(m.PrometheusDataSourceName), stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationVertical), + stat.Orientation(stat.OrientationAuto), stat.TitleFontSize(12), stat.ValueFontSize(20), stat.Span(2), stat.Text("name"), - stat.Height("100px"), stat.WithPrometheusTarget( - `version{`+m.panelOption.labelFilter+`=~"$instance"}`, + `version{`+m.panelOption.labelQuery+`}`, prometheus.Legend("{{version}}"), ), ), @@ -41,14 +34,13 @@ func (m *Dashboard) addMainPanels() { "Go Version", stat.DataSource(m.PrometheusDataSourceName), stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationVertical), + stat.Orientation(stat.OrientationAuto), stat.TitleFontSize(12), stat.ValueFontSize(20), stat.Span(2), stat.Text("name"), - stat.Height("100px"), stat.WithPrometheusTarget( - `go_info{`+m.panelOption.labelFilter+`=~"$instance"}`, + `go_info{`+m.panelOption.labelQuery+`}`, prometheus.Legend("{{"+goVersionLegend+"}}"), ), ), @@ -56,43 +48,40 @@ func (m *Dashboard) addMainPanels() { "Uptime in days", stat.DataSource(m.PrometheusDataSourceName), stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationVertical), + stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), stat.ValueFontSize(20), - stat.Span(2), - stat.Height("100px"), + stat.Span(8), stat.WithPrometheusTarget( - `uptime_seconds{`+m.panelOption.labelFilter+`=~"$instance"} / 86400`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `uptime_seconds{`+m.panelOption.labelQuery+`} / 86400`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithStat( "ETH Balance", stat.DataSource(m.PrometheusDataSourceName), stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationVertical), + stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), stat.ValueFontSize(20), - stat.Span(balancePanelSpanSize), - stat.Height("100px"), + stat.Span(6), stat.Decimals(2), stat.WithPrometheusTarget( - `eth_balance{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{account}}"), + `eth_balance{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{account}}"), ), ), row.WithStat( "Solana Balance", stat.DataSource(m.PrometheusDataSourceName), stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationVertical), + stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), stat.ValueFontSize(20), - stat.Span(balancePanelSpanSize), - stat.Height("100px"), + stat.Span(6), stat.Decimals(2), stat.WithPrometheusTarget( - `solana_balance{`+m.panelOption.labelFilter+`=~"$instance"}`, + `solana_balance{`+m.panelOption.labelQuery+`}`, prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{account}}"), ), ), @@ -105,8 +94,8 @@ func (m *Dashboard) addMainPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `health{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{service_id}}"), + `health{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{service_id}}"), ), ), row.WithTimeSeries( @@ -119,8 +108,8 @@ func (m *Dashboard) addMainPanels() { axis.Decimals(2), ), timeseries.WithPrometheusTarget( - `eth_balance{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{account}}"), + `eth_balance{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{account}}"), ), ), row.WithTimeSeries( @@ -133,8 +122,8 @@ func (m *Dashboard) addMainPanels() { axis.Decimals(2), ), timeseries.WithPrometheusTarget( - `solana_balance{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{account}}"), + `solana_balance{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{account}}"), ), ), } @@ -171,36 +160,39 @@ func (m *Dashboard) addKubePanels() { row.WithStat( "Pod Restarts", stat.Span(4), - stat.Height("200px"), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), stat.DataSource(m.PrometheusDataSourceName), stat.SparkLine(), stat.SparkLineYMin(0), stat.WithPrometheusTarget( - `sum(increase(kube_pod_container_status_restarts_total{pod=~"$instance.*", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, + `sum(increase(kube_pod_container_status_restarts_total{pod=~"$pod", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, prometheus.Legend("{{pod}}"), ), ), row.WithStat( "OOM Events", stat.Span(4), - stat.Height("200px"), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), stat.DataSource(m.PrometheusDataSourceName), stat.SparkLine(), stat.SparkLineYMin(0), stat.WithPrometheusTarget( - `sum(container_oom_events_total{pod=~"$instance.*", namespace=~"${namespace}"}) by (pod)`, + `sum(container_oom_events_total{pod=~"$pod", namespace=~"${namespace}"}) by (pod)`, prometheus.Legend("{{pod}}"), ), ), row.WithStat( "OOM Killed", stat.Span(4), - stat.Height("200px"), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), stat.DataSource(m.PrometheusDataSourceName), stat.SparkLine(), stat.SparkLineYMin(0), stat.WithPrometheusTarget( - `kube_pod_container_status_last_terminated_reason{reason="OOMKilled", pod=~"$instance.*", namespace=~"${namespace}"}`, + `kube_pod_container_status_last_terminated_reason{reason="OOMKilled", pod=~"$pod", namespace=~"${namespace}"}`, prometheus.Legend("{{pod}}"), ), ), @@ -210,7 +202,7 @@ func (m *Dashboard) addKubePanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~"$instance.*", namespace=~"${namespace}"}) by (pod)`, + `sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~"$pod", namespace=~"${namespace}"}) by (pod)`, prometheus.Legend("{{pod}}"), ), ), @@ -225,7 +217,7 @@ func (m *Dashboard) addKubePanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `sum(container_memory_rss{pod=~"$instance.*", namespace=~"${namespace}", container!=""}) by (pod)`, + `sum(container_memory_rss{pod=~"$pod", namespace=~"${namespace}", container!=""}) by (pod)`, prometheus.Legend("{{pod}}"), ), ), @@ -239,7 +231,7 @@ func (m *Dashboard) addKubePanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `sum(irate(container_network_receive_bytes_total{pod=~"$instance.*", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, + `sum(irate(container_network_receive_bytes_total{pod=~"$pod", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, prometheus.Legend("{{pod}}"), ), ), @@ -253,7 +245,7 @@ func (m *Dashboard) addKubePanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `sum(irate(container_network_transmit_bytes_total{pod=~"$instance.*", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, + `sum(irate(container_network_transmit_bytes_total{pod=~"$pod", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, prometheus.Legend("{{pod}}"), ), ), @@ -267,7 +259,7 @@ func (m *Dashboard) addKubePanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `avg(irate(container_network_receive_bytes_total{pod=~"$instance.*", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, + `avg(irate(container_network_receive_bytes_total{pod=~"$pod", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, prometheus.Legend("{{pod}}"), ), ), @@ -281,7 +273,7 @@ func (m *Dashboard) addKubePanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `avg(irate(container_network_transmit_bytes_total{pod=~"$instance.*", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, + `avg(irate(container_network_transmit_bytes_total{pod=~"$pod", namespace=~"${namespace}"}[$__rate_interval])) by (pod)`, prometheus.Legend("{{pod}}"), ), ), @@ -301,11 +293,11 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `avg(sum(rate(log_poller_query_duration_count{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (query, instance)) by (query)`, + `avg(sum(rate(log_poller_query_duration_count{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}[$__rate_interval])) by (query, `+m.panelOption.labelFilter+`)) by (query)`, prometheus.Legend("{{query}}"), ), timeseries.WithPrometheusTarget( - `avg(sum(rate(log_poller_query_duration_count{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval]))) by (instance)`, + `avg(sum(rate(log_poller_query_duration_count{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval]))) by (`+m.panelOption.labelFilter+`)`, prometheus.Legend("Total"), ), ), @@ -315,7 +307,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `log_poller_query_dataset_size{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}`, + `log_poller_query_dataset_size{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}`, prometheus.Legend("{{query}} : {{type}}"), ), ), @@ -325,7 +317,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `avg(log_poller_query_dataset_size{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}) by (query)`, + `avg(log_poller_query_dataset_size{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}) by (query)`, prometheus.Legend("{{query}}"), ), ), @@ -335,7 +327,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `max(log_poller_query_dataset_size{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}) by (query)`, + `max(log_poller_query_dataset_size{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}) by (query)`, prometheus.Legend("{{query}}"), ), ), @@ -345,7 +337,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `max(log_poller_query_dataset_size{`+m.panelOption.labelFilter+`=~"$instance"}) by (evmChainID)`, + `max(log_poller_query_dataset_size{`+m.panelOption.labelQuery+`}) by (evmChainID)`, prometheus.Legend("{{evmChainID}}"), ), ), @@ -355,7 +347,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `(sum(rate(log_poller_query_duration_sum{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (query) / sum(rate(log_poller_query_duration_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (query)) / 1e6`, + `(sum(rate(log_poller_query_duration_sum{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}[$__rate_interval])) by (query) / sum(rate(log_poller_query_duration_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (query)) / 1e6`, prometheus.Legend("{{query}}"), ), ), @@ -365,7 +357,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `histogram_quantile(0.99, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, + `histogram_quantile(0.99, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, prometheus.Legend("{{query}}"), ), ), @@ -375,7 +367,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `histogram_quantile(0.95, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, + `histogram_quantile(0.95, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, prometheus.Legend("{{query}}"), ), ), @@ -385,7 +377,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `histogram_quantile(0.95, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, + `histogram_quantile(0.95, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, prometheus.Legend("{{query}}"), ), ), @@ -395,7 +387,7 @@ func (m *Dashboard) addLogPollerPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `histogram_quantile(0.5, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelFilter+`=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, + `histogram_quantile(0.5, sum(rate(log_poller_query_duration_bucket{`+m.panelOption.labelQuery+`evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`, prometheus.Legend("{{query}}"), ), ), @@ -418,8 +410,8 @@ func (m *Dashboard) addFeedsJobsPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `feeds_job_proposal_requests{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `feeds_job_proposal_requests{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -431,8 +423,8 @@ func (m *Dashboard) addFeedsJobsPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `feeds_job_proposal_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `feeds_job_proposal_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -454,8 +446,8 @@ func (m *Dashboard) addMailboxPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `mailbox_load_percent{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{ name }}"), + `mailbox_load_percent{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{ name }}"), ), ), ), @@ -477,8 +469,8 @@ func (m *Dashboard) addPromReporterPanels() { axis.Unit("Tx"), ), timeseries.WithPrometheusTarget( - `unconfirmed_transactions{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `unconfirmed_transactions{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -490,8 +482,8 @@ func (m *Dashboard) addPromReporterPanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `max_unconfirmed_tx_age{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `max_unconfirmed_tx_age{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -503,8 +495,8 @@ func (m *Dashboard) addPromReporterPanels() { axis.Unit("Blocks"), ), timeseries.WithPrometheusTarget( - `max_unconfirmed_blocks{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `max_unconfirmed_blocks{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -526,8 +518,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_time_until_tx_broadcast{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_time_until_tx_broadcast{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -539,8 +531,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_num_gas_bumps{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_num_gas_bumps{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -552,8 +544,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_gas_bump_exceeds_limit{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_gas_bump_exceeds_limit{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -565,8 +557,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_num_confirmed_transactions{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_num_confirmed_transactions{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -578,8 +570,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_num_successful_transactions{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_num_successful_transactions{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -591,8 +583,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_num_tx_reverted{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_num_tx_reverted{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -604,8 +596,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_fwd_tx_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_fwd_tx_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -617,8 +609,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_tx_attempt_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_tx_attempt_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -630,8 +622,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_time_until_tx_confirmed{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_time_until_tx_confirmed{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -643,8 +635,8 @@ func (m *Dashboard) addTxManagerPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `tx_manager_blocks_until_tx_confirmed{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `tx_manager_blocks_until_tx_confirmed{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -666,8 +658,8 @@ func (m *Dashboard) addHeadTrackerPanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `head_tracker_current_head{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `head_tracker_current_head{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -679,8 +671,8 @@ func (m *Dashboard) addHeadTrackerPanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `head_tracker_very_old_head{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `head_tracker_very_old_head{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -692,8 +684,8 @@ func (m *Dashboard) addHeadTrackerPanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `head_tracker_heads_received{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `head_tracker_heads_received{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -705,8 +697,8 @@ func (m *Dashboard) addHeadTrackerPanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `head_tracker_connection_errors{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `head_tracker_connection_errors{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -729,20 +721,20 @@ func (m *Dashboard) addDatabasePanels() { axis.Unit("Conn"), ), timeseries.WithPrometheusTarget( - `db_conns_max{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Max"), + `db_conns_max{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Max"), ), timeseries.WithPrometheusTarget( - `db_conns_open{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Open"), + `db_conns_open{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Open"), ), timeseries.WithPrometheusTarget( - `db_conns_used{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Used"), + `db_conns_used{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Used"), ), timeseries.WithPrometheusTarget( - `db_conns_wait{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Wait"), + `db_conns_wait{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Wait"), ), ), row.WithTimeSeries( @@ -754,8 +746,8 @@ func (m *Dashboard) addDatabasePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `db_wait_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `db_wait_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -767,8 +759,8 @@ func (m *Dashboard) addDatabasePanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `db_wait_time_seconds{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `db_wait_time_seconds{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -791,15 +783,15 @@ func (m *Dashboard) addSQLQueryPanels() { axis.Unit("percent"), ), timeseries.WithPrometheusTarget( - `histogram_quantile(0.9, sum(rate(sql_query_timeout_percent_bucket{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (le))`, + `histogram_quantile(0.9, sum(rate(sql_query_timeout_percent_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (le))`, prometheus.Legend("p90"), ), timeseries.WithPrometheusTarget( - `histogram_quantile(0.95, sum(rate(sql_query_timeout_percent_bucket{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (le))`, + `histogram_quantile(0.95, sum(rate(sql_query_timeout_percent_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (le))`, prometheus.Legend("p95"), ), timeseries.WithPrometheusTarget( - `histogram_quantile(0.99, sum(rate(sql_query_timeout_percent_bucket{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (le))`, + `histogram_quantile(0.99, sum(rate(sql_query_timeout_percent_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (le))`, prometheus.Legend("p99"), ), ), @@ -819,24 +811,24 @@ func (m *Dashboard) addLogsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `log_panic_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - panic"), + `log_panic_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - panic"), ), timeseries.WithPrometheusTarget( - `log_fatal_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - fatal"), + `log_fatal_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - fatal"), ), timeseries.WithPrometheusTarget( - `log_critical_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - critical"), + `log_critical_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - critical"), ), timeseries.WithPrometheusTarget( - `log_warn_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - warn"), + `log_warn_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - warn"), ), timeseries.WithPrometheusTarget( - `log_error_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - error"), + `log_error_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - error"), ), ), row.WithTimeSeries( @@ -845,24 +837,24 @@ func (m *Dashboard) addLogsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `sum(rate(log_panic_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("panic"), + `sum(rate(log_panic_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - panic"), ), timeseries.WithPrometheusTarget( - `sum(rate(log_fatal_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("fatal"), + `sum(rate(log_fatal_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - fatal"), ), timeseries.WithPrometheusTarget( - `sum(rate(log_critical_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("critical"), + `sum(rate(log_critical_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - critical"), ), timeseries.WithPrometheusTarget( - `sum(rate(log_warn_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("warn"), + `sum(rate(log_warn_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - warn"), ), timeseries.WithPrometheusTarget( - `sum(rate(log_error_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("error"), + `sum(rate(log_error_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - error"), ), ), ), @@ -885,8 +877,8 @@ func (m *Dashboard) addEVMPoolLifecyclePanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_highest_seen_block{`+m.panelOption.labelFilter+`=~"$instance", evmChainID="${evmChainID}"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `evm_pool_rpc_node_highest_seen_block{`+m.panelOption.labelQuery+`evmChainID="${evmChainID}"}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -898,8 +890,8 @@ func (m *Dashboard) addEVMPoolLifecyclePanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_seen_blocks{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `evm_pool_rpc_node_num_seen_blocks{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -911,8 +903,8 @@ func (m *Dashboard) addEVMPoolLifecyclePanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_polls_total{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `evm_pool_rpc_node_polls_total{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -924,8 +916,8 @@ func (m *Dashboard) addEVMPoolLifecyclePanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_polls_failed{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `evm_pool_rpc_node_polls_failed{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -937,8 +929,8 @@ func (m *Dashboard) addEVMPoolLifecyclePanels() { axis.Unit("Block"), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_polls_success{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `evm_pool_rpc_node_polls_success{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -954,7 +946,7 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { row.Collapse(), row.WithTimeSeries( "EVM Pool RPC Node Calls Success Rate", - timeseries.Span(7), + timeseries.Span(12), timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.Axis( @@ -964,19 +956,19 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.SoftMax(100), ), timeseries.WithPrometheusTarget( - `sum(increase(evm_pool_rpc_node_calls_success{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_calls_total{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) * 100`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{nodeName}}"), + `sum(increase(evm_pool_rpc_node_calls_success{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_calls_total{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) * 100`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{nodeName}}"), ), ), row.WithGauge( "EVM Pool RPC Node Calls Success Rate", - gauge.Span(5), - gauge.Height("200px"), + gauge.Span(12), + gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(m.PrometheusDataSourceName), gauge.Unit("percentunit"), gauge.WithPrometheusTarget( - `sum(increase(evm_pool_rpc_node_calls_success{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_calls_total{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName)`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{nodeName}}"), + `sum(increase(evm_pool_rpc_node_calls_success{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_calls_total{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{nodeName}}"), ), gauge.AbsoluteThresholds([]gauge.ThresholdStep{ {Color: "#ff0000"}, @@ -997,8 +989,8 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.SoftMax(100), ), timeseries.WithPrometheusTarget( - `sum(increase(evm_pool_rpc_node_dials_success{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_dials_total{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) * 100`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{nodeName}}"), + `sum(increase(evm_pool_rpc_node_dials_success{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_dials_total{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) * 100`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{nodeName}}"), ), ), // issue when value is 0 @@ -1014,8 +1006,8 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.SoftMax(100), ), timeseries.WithPrometheusTarget( - `sum(increase(evm_pool_rpc_node_dials_failed{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_dials_total{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) * 100`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{nodeName}}"), + `sum(increase(evm_pool_rpc_node_dials_failed{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_dials_total{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) * 100`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{nodeName}}"), ), ), row.WithTimeSeries( @@ -1027,27 +1019,27 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_transitions_to_alive{`+m.panelOption.labelFilter+`=~"$instance"}`, + `evm_pool_rpc_node_num_transitions_to_alive{`+m.panelOption.labelQuery+`}`, prometheus.Legend(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_transitions_to_in_sync{`+m.panelOption.labelFilter+`=~"$instance"}`, + `evm_pool_rpc_node_num_transitions_to_in_sync{`+m.panelOption.labelQuery+`}`, prometheus.Legend(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_transitions_to_out_of_sync{`+m.panelOption.labelFilter+`=~"$instance"}`, + `evm_pool_rpc_node_num_transitions_to_out_of_sync{`+m.panelOption.labelQuery+`}`, prometheus.Legend(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_transitions_to_unreachable{`+m.panelOption.labelFilter+`=~"$instance"}`, + `evm_pool_rpc_node_num_transitions_to_unreachable{`+m.panelOption.labelQuery+`}`, prometheus.Legend(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_transitions_to_invalid_chain_id{`+m.panelOption.labelFilter+`=~"$instance"}`, + `evm_pool_rpc_node_num_transitions_to_invalid_chain_id{`+m.panelOption.labelQuery+`}`, prometheus.Legend(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_num_transitions_to_unusable{`+m.panelOption.labelFilter+`=~"$instance"}`, + `evm_pool_rpc_node_num_transitions_to_unusable{`+m.panelOption.labelQuery+`}`, prometheus.Legend(""), ), ), @@ -1060,8 +1052,8 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `evm_pool_rpc_node_states{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{state}}"), + `evm_pool_rpc_node_states{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{state}}"), ), ), row.WithTimeSeries( @@ -1076,8 +1068,8 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.SoftMax(100), ), timeseries.WithPrometheusTarget( - `sum(increase(evm_pool_rpc_node_verifies_success{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_verifies{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) * 100`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{nodeName}}"), + `sum(increase(evm_pool_rpc_node_verifies_success{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_verifies{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) * 100`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{nodeName}}"), ), ), row.WithTimeSeries( @@ -1092,8 +1084,8 @@ func (m *Dashboard) addEVMPoolRPCNodePanels() { axis.SoftMax(100), ), timeseries.WithPrometheusTarget( - `sum(increase(evm_pool_rpc_node_verifies_failed{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_verifies{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (`+m.panelOption.labelFilter+`, evmChainID, nodeName) * 100`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{evmChainID}} - {{nodeName}}"), + `sum(increase(evm_pool_rpc_node_verifies_failed{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) / sum(increase(evm_pool_rpc_node_verifies{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, evmChainID, nodeName) * 100`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{evmChainID}} - {{nodeName}}"), ), ), ), @@ -1107,17 +1099,43 @@ func (m *Dashboard) addEVMRPCNodeLatenciesPanels() { dashboard.Row( "EVM Pool RPC Node Latencies (App)", row.Collapse(), + row.WithTimeSeries( + "EVM Pool RPC Node Calls Latency 0.90 quantile", + timeseries.Span(12), + timeseries.Height("200px"), + timeseries.DataSource(m.PrometheusDataSourceName), + timeseries.Axis( + axis.Unit("ms"), + ), + timeseries.WithPrometheusTarget( + `histogram_quantile(0.90, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, le, rpcCallName)) / 1e6`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{rpcCallName}}"), + ), + ), row.WithTimeSeries( "EVM Pool RPC Node Calls Latency 0.95 quantile", - timeseries.Span(6), + timeseries.Span(12), timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.Axis( axis.Unit("ms"), ), timeseries.WithPrometheusTarget( - `histogram_quantile(0.95, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (le, rpcCallName)) / 1e6`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `histogram_quantile(0.95, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, le, rpcCallName)) / 1e6`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{rpcCallName}}"), + ), + ), + row.WithTimeSeries( + "EVM Pool RPC Node Calls Latency 0.99 quantile", + timeseries.Span(12), + timeseries.Height("200px"), + timeseries.DataSource(m.PrometheusDataSourceName), + timeseries.Axis( + axis.Unit("ms"), + ), + timeseries.WithPrometheusTarget( + `histogram_quantile(0.99, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, le, rpcCallName)) / 1e6`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{rpcCallName}}"), ), ), ), @@ -1139,8 +1157,8 @@ func (m *Dashboard) addBlockHistoryEstimatorPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `gas_updater_all_gas_price_percentiles{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{ percentile }}"), + `gas_updater_all_gas_price_percentiles{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{ percentile }}"), ), ), row.WithTimeSeries( @@ -1152,8 +1170,8 @@ func (m *Dashboard) addBlockHistoryEstimatorPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `gas_updater_all_tip_cap_percentiles{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - {{ percentile }}"), + `gas_updater_all_tip_cap_percentiles{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{ percentile }}"), ), ), row.WithTimeSeries( @@ -1165,8 +1183,8 @@ func (m *Dashboard) addBlockHistoryEstimatorPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `gas_updater_set_gas_price{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `gas_updater_set_gas_price{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1178,8 +1196,8 @@ func (m *Dashboard) addBlockHistoryEstimatorPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `gas_updater_set_tip_cap{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `gas_updater_set_tip_cap{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1191,8 +1209,8 @@ func (m *Dashboard) addBlockHistoryEstimatorPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `gas_updater_current_base_fee{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `gas_updater_current_base_fee{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1204,8 +1222,8 @@ func (m *Dashboard) addBlockHistoryEstimatorPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `block_history_estimator_connectivity_failure_count{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `block_history_estimator_connectivity_failure_count{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1227,8 +1245,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `pipeline_task_execution_time{`+m.panelOption.labelFilter+`=~"$instance"} / 1e6`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} JobID: {{ job_id }}"), + `pipeline_task_execution_time{`+m.panelOption.labelQuery+`} / 1e6`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} JobID: {{ job_id }}"), ), ), row.WithTimeSeries( @@ -1240,8 +1258,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `pipeline_run_errors{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} JobID: {{ job_id }}"), + `pipeline_run_errors{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} JobID: {{ job_id }}"), ), ), row.WithTimeSeries( @@ -1253,8 +1271,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `pipeline_run_total_time_to_completion{`+m.panelOption.labelFilter+`=~"$instance"} / 1e6`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} JobID: {{ job_id }}"), + `pipeline_run_total_time_to_completion{`+m.panelOption.labelQuery+`} / 1e6`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} JobID: {{ job_id }}"), ), ), row.WithTimeSeries( @@ -1266,8 +1284,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `pipeline_tasks_total_finished{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} JobID: {{ job_id }}"), + `pipeline_tasks_total_finished{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} JobID: {{ job_id }}"), ), ), ), @@ -1283,8 +1301,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `pipeline_task_eth_call_execution_time{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `pipeline_task_eth_call_execution_time{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1300,8 +1318,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `pipeline_task_http_fetch_time{`+m.panelOption.labelFilter+`=~"$instance"} / 1e6`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `pipeline_task_http_fetch_time{`+m.panelOption.labelQuery+`} / 1e6`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1313,8 +1331,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit("Bytes"), ), timeseries.WithPrometheusTarget( - `pipeline_task_http_response_body_size{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `pipeline_task_http_response_body_size{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1330,8 +1348,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `bridge_latency_seconds{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `bridge_latency_seconds{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1343,8 +1361,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `bridge_errors_total{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `bridge_errors_total{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1356,8 +1374,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `bridge_cache_hits_total{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `bridge_cache_hits_total{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1369,8 +1387,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `bridge_cache_errors_total{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `bridge_cache_errors_total{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1386,8 +1404,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `pipeline_runs_queued{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `pipeline_runs_queued{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1399,8 +1417,8 @@ func (m *Dashboard) addPipelinePanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `pipeline_task_runs_queued{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `pipeline_task_runs_queued{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1424,8 +1442,8 @@ func (m *Dashboard) addHTTPAPIPanels() { axis.Unit("Sec"), ), timeseries.WithPrometheusTarget( - `histogram_quantile(0.95, sum(rate(service_gonic_request_duration_bucket{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (le, path, method))`, - prometheus.Legend("{{ method }} {{ path }}"), + `histogram_quantile(0.95, sum(rate(service_gonic_request_duration_bucket{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, le, path, method))`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{ method }} - {{ path }}"), ), ), row.WithTimeSeries( @@ -1437,12 +1455,12 @@ func (m *Dashboard) addHTTPAPIPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `sum(rate(service_gonic_requests_total{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (path, method, code)`, - prometheus.Legend("{{ method }} {{ path }} {{ code }}"), + `sum(rate(service_gonic_requests_total{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, path, method, code)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - {{ method }} - {{ path }} - {{ code }}"), ), ), row.WithTimeSeries( - "Request Size", + "Average Request Size", timeseries.Span(6), timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), @@ -1450,8 +1468,8 @@ func (m *Dashboard) addHTTPAPIPanels() { axis.Unit("Bytes"), ), timeseries.WithPrometheusTarget( - `avg(rate(service_gonic_request_size_bytes_sum{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))/avg(rate(service_gonic_request_size_bytes_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("Average"), + `avg(rate(service_gonic_request_size_bytes_sum{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)/avg(rate(service_gonic_request_size_bytes_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( @@ -1463,8 +1481,8 @@ func (m *Dashboard) addHTTPAPIPanels() { axis.Unit("Bytes"), ), timeseries.WithPrometheusTarget( - `avg(rate(service_gonic_response_size_bytes_sum{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))/avg(rate(service_gonic_response_size_bytes_count{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval]))`, - prometheus.Legend("Average"), + `avg(rate(service_gonic_response_size_bytes_sum{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)/avg(rate(service_gonic_response_size_bytes_count{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1479,25 +1497,25 @@ func (m *Dashboard) addPromHTTPPanels() { "PromHTTP Metrics", row.Collapse(), row.WithGauge("HTTP Request in flight", - gauge.Span(2), - gauge.Height("200px"), + gauge.Span(12), + gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(m.PrometheusDataSourceName), gauge.WithPrometheusTarget( - `promhttp_metric_handler_requests_in_flight`, - prometheus.Legend(""), + `promhttp_metric_handler_requests_in_flight{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), row.WithTimeSeries( "HTTP rate", - timeseries.Span(10), + timeseries.Span(12), timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.Axis( axis.Unit(""), ), timeseries.WithPrometheusTarget( - `sum(rate(promhttp_metric_handler_requests_total{`+m.panelOption.labelFilter+`=~"$instance"}[$__rate_interval])) by (code)`, - prometheus.Legend("{{ code }}"), + `sum(rate(promhttp_metric_handler_requests_total{`+m.panelOption.labelQuery+`}[$__rate_interval])) by (`+m.panelOption.legendString+`, code)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), ), ), @@ -1517,8 +1535,8 @@ func (m *Dashboard) addGoMetricsPanels() { table.Height("200px"), table.DataSource(m.PrometheusDataSourceName), table.WithPrometheusTarget( - `sum(go_threads{`+m.panelOption.labelFilter+`=~"$instance"}) by (instance)`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}")), + `sum(go_threads{`+m.panelOption.labelQuery+`}) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}")), table.HideColumn("Time"), table.AsTimeSeriesAggregations([]table.Aggregation{ {Label: "AVG", Type: table.AVG}, @@ -1534,18 +1552,29 @@ func (m *Dashboard) addGoMetricsPanels() { axis.Unit(""), ), timeseries.WithPrometheusTarget( - `sum(go_threads{`+m.panelOption.labelFilter+`=~"$instance"}) by (instance)`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `sum(go_threads{`+m.panelOption.labelQuery+`}) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), + ), + ), + row.WithStat( + "Heap Allocations", + stat.Span(12), + stat.Orientation(stat.OrientationVertical), + stat.DataSource(m.PrometheusDataSourceName), + stat.Unit("bytes"), + stat.ColorValue(), + stat.WithPrometheusTarget( + `sum(go_memstats_heap_alloc_bytes{`+m.panelOption.labelQuery+`}) by (`+m.panelOption.legendString+`)`, ), ), row.WithTimeSeries( "Heap allocations", - timeseries.Span(6), + timeseries.Span(12), timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `sum(go_memstats_heap_alloc_bytes{`+m.panelOption.labelFilter+`=~"$instance"}) by (instance)`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `sum(go_memstats_heap_alloc_bytes{`+m.panelOption.labelQuery+`}) by (`+m.panelOption.legendString+`)`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), timeseries.Axis( axis.Unit("bytes"), @@ -1553,29 +1582,6 @@ func (m *Dashboard) addGoMetricsPanels() { axis.SoftMin(0), ), ), - row.WithStat( - "Heap Allocations", - stat.Span(6), - stat.Height("200px"), - stat.DataSource(m.PrometheusDataSourceName), - stat.Unit("bytes"), - stat.ColorValue(), - stat.WithPrometheusTarget(`sum(go_memstats_heap_alloc_bytes{`+m.panelOption.labelFilter+`=~"$instance"})`), - /*stat.AbsoluteThresholds([]stat.ThresholdStep{ - { - Color: "green", - Value: nil, - }, - { - Color: "orange", - Value: float64Ptr(6.711e+7), - }, - { - Color: "red", - Value: float64Ptr(1.342e+8), - }, - }),*/ - ), row.WithTimeSeries( "Memory in Heap", timeseries.Span(6), @@ -1587,24 +1593,24 @@ func (m *Dashboard) addGoMetricsPanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `go_memstats_heap_alloc_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Alloc"), + `go_memstats_heap_alloc_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Alloc"), ), timeseries.WithPrometheusTarget( - `go_memstats_heap_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Sys"), + `go_memstats_heap_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Sys"), ), timeseries.WithPrometheusTarget( - `go_memstats_heap_idle_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Idle"), + `go_memstats_heap_idle_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Idle"), ), timeseries.WithPrometheusTarget( - `go_memstats_heap_inuse_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - InUse"), + `go_memstats_heap_inuse_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - InUse"), ), timeseries.WithPrometheusTarget( - `go_memstats_heap_released_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Released"), + `go_memstats_heap_released_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Released"), ), ), row.WithTimeSeries( @@ -1618,36 +1624,36 @@ func (m *Dashboard) addGoMetricsPanels() { axis.SoftMin(0), ), timeseries.WithPrometheusTarget( - `go_memstats_mspan_inuse_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Total InUse"), + `go_memstats_mspan_inuse_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Total InUse"), ), timeseries.WithPrometheusTarget( - `go_memstats_mspan_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Total Sys"), + `go_memstats_mspan_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Total Sys"), ), timeseries.WithPrometheusTarget( - `go_memstats_mcache_inuse_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Cache InUse"), + `go_memstats_mcache_inuse_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Cache InUse"), ), timeseries.WithPrometheusTarget( - `go_memstats_mcache_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Cache Sys"), + `go_memstats_mcache_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Cache Sys"), ), timeseries.WithPrometheusTarget( - `go_memstats_buck_hash_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Hash Sys"), + `go_memstats_buck_hash_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Hash Sys"), ), timeseries.WithPrometheusTarget( - `go_memstats_gc_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - GC Sys"), + `go_memstats_gc_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - GC Sys"), ), timeseries.WithPrometheusTarget( - `go_memstats_other_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - bytes of memory are used for other runtime allocations"), + `go_memstats_other_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - bytes of memory are used for other runtime allocations"), ), timeseries.WithPrometheusTarget( - `go_memstats_next_gc_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Next GC"), + `go_memstats_next_gc_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Next GC"), ), ), row.WithTimeSeries( @@ -1656,12 +1662,12 @@ func (m *Dashboard) addGoMetricsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `go_memstats_stack_inuse_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - InUse"), + `go_memstats_stack_inuse_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - InUse"), ), timeseries.WithPrometheusTarget( - `go_memstats_stack_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}} - Sys"), + `go_memstats_stack_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}} - Sys"), ), timeseries.Axis( axis.Unit("bytes"), @@ -1675,8 +1681,8 @@ func (m *Dashboard) addGoMetricsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `go_memstats_sys_bytes{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `go_memstats_sys_bytes{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), timeseries.Axis( axis.Unit("bytes"), @@ -1690,8 +1696,8 @@ func (m *Dashboard) addGoMetricsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `go_memstats_mallocs_total{`+m.panelOption.labelFilter+`=~"$instance"} - go_memstats_frees_total{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `go_memstats_mallocs_total{`+m.panelOption.labelQuery+`} - go_memstats_frees_total{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), timeseries.Axis( axis.SoftMin(0), @@ -1703,8 +1709,8 @@ func (m *Dashboard) addGoMetricsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `rate(go_memstats_mallocs_total{`+m.panelOption.labelFilter+`=~"$instance"}[1m])`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `rate(go_memstats_mallocs_total{`+m.panelOption.labelQuery+`}[1m])`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), timeseries.Axis( axis.SoftMin(0), @@ -1716,8 +1722,8 @@ func (m *Dashboard) addGoMetricsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `rate(go_memstats_lookups_total{`+m.panelOption.labelFilter+`=~"$instance"}[1m])`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `rate(go_memstats_lookups_total{`+m.panelOption.labelQuery+`}[1m])`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), timeseries.Axis( axis.Unit("ops"), @@ -1730,8 +1736,8 @@ func (m *Dashboard) addGoMetricsPanels() { timeseries.Height("200px"), timeseries.DataSource(m.PrometheusDataSourceName), timeseries.WithPrometheusTarget( - `go_goroutines{`+m.panelOption.labelFilter+`=~"$instance"}`, - prometheus.Legend("{{"+m.panelOption.labelFilter+"}}"), + `go_goroutines{`+m.panelOption.labelQuery+`}`, + prometheus.Legend("{{"+m.panelOption.legendString+"}}"), ), timeseries.Axis( axis.SoftMin(0),