From 8698e76ba8de6013e6e74bc6017977dff4873fd1 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Wed, 13 Mar 2024 15:41:17 +0100 Subject: [PATCH 01/11] feat(dashboard): add DON ocr contract panels --- .../chainlink-cluster/dashboard/cmd/deploy.go | 14 +- dashboard/lib/atlas-don/component.go | 341 ++++++++++++++++++ dashboard/lib/atlas-don/platform.go | 49 +++ 3 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 dashboard/lib/atlas-don/component.go create mode 100644 dashboard/lib/atlas-don/platform.go diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index 9eb89ae3bd2..ffb5397b370 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -3,13 +3,14 @@ package main import ( "github.com/K-Phoen/grabana/dashboard" lib "github.com/smartcontractkit/chainlink/dashboard-lib/lib" + atlas_don "github.com/smartcontractkit/chainlink/dashboard-lib/lib/atlas-don" core_don "github.com/smartcontractkit/chainlink/dashboard-lib/lib/core-don" k8spods "github.com/smartcontractkit/chainlink/dashboard-lib/lib/k8s-pods" waspdb "github.com/smartcontractkit/wasp/dashboard" ) const ( - DashboardName = "Chainlink Cluster (DON)" + DashboardName = "ClementChainlinkCluster" ) func main() { @@ -38,8 +39,17 @@ func main() { ), ) } + db.Add( + atlas_don.New( + atlas_don.Props{ + PrometheusDataSource: cfg.DataSources.Prometheus, + PlatformOpts: atlas_don.PlatformPanelOpts(cfg.Platform), + OcrVersion: "ocr", + }, + ), + ) // TODO: refactor as a component later - addWASPRows(db, cfg) + // addWASPRows(db, cfg) if err := db.Deploy(); err != nil { lib.L.Fatal().Err(err).Msg("failed to deploy the dashboard") } diff --git a/dashboard/lib/atlas-don/component.go b/dashboard/lib/atlas-don/component.go new file mode 100644 index 00000000000..71ba8e965bb --- /dev/null +++ b/dashboard/lib/atlas-don/component.go @@ -0,0 +1,341 @@ +package atlas_don + +import ( + "fmt" + "github.com/K-Phoen/grabana/dashboard" + "github.com/K-Phoen/grabana/gauge" + "github.com/K-Phoen/grabana/row" + "github.com/K-Phoen/grabana/stat" + "github.com/K-Phoen/grabana/target/prometheus" + "github.com/K-Phoen/grabana/variable/query" +) + +type Props struct { + PrometheusDataSource string + PlatformOpts PlatformOpts + OcrVersion string +} + +func vars(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.VariableAsQuery( + "contract", + query.DataSource(p.PrometheusDataSource), + query.Multiple(), + query.IncludeAll(), + query.Request(fmt.Sprintf("label_values(%s)", "contract")), + query.Sort(query.NumericalAsc), + ), + } +} + +func ocrContractConfigOracle(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("OCR Contract Oracle", + row.Collapse(), + row.WithStat( + "OCR Contract Oracle Active", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Description("set to one as long as an oracle is on a feed"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(12), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }} - {{oracle}}"), + ), + ), + row.WithGauge( + "OCR Contract Oracle Active", + gauge.Span(12), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.Description("set to one as long as an oracle is on a feed"), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }} - {{oracle}}"), + ), + ), + ), + } +} + +func ocrContractConfigNodes(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("OCR Contract Config Nodes", + row.Collapse(), + row.WithStat( + "Node Count", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "Node Count", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "Max Faulty Node Count", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "Max Faulty Node Count", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "Max Round Node Count", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "Max Round Node Count", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + ), + } +} + +func ocrContractConfigDelta(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("OCR Contract Config Delta", + row.Collapse(), + row.WithStat( + "relativeDeviationThreshold", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "relativeDeviationThreshold", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "maxContractValueAgeSeconds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "maxContractValueAgeSeconds", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "observationGracePeriodSeconds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "observationGracePeriodSeconds", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "badEpochTimeoutSeconds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "badEpochTimeoutSeconds", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "resendIntervalSeconds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "resendIntervalSeconds", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "roundIntervalSeconds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "roundIntervalSeconds", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithStat( + "transmissionStageTimeoutSeconds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(20), + stat.Span(6), + stat.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + row.WithGauge( + "transmissionStageTimeoutSeconds", + gauge.Span(6), + gauge.Orientation(gauge.OrientationVertical), + gauge.DataSource(p.PrometheusDataSource), + gauge.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + ), + ), + ), + } +} + +func ocrTelemetryFeed(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("OCR Telemetry Feed", + row.Collapse(), + ), + } +} + +func ocrTelemetryP2P(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("OCR Telemetry P2P", + row.Collapse(), + ), + } +} + +func ocrTelemetryOthers(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("OCR Telemetry Others", + row.Collapse(), + ), + } +} + +func New(p Props) []dashboard.Option { + opts := vars(p) + opts = append(opts, ocrContractConfigOracle(p)...) + opts = append(opts, ocrContractConfigNodes(p)...) + opts = append(opts, ocrContractConfigDelta(p)...) + return opts +} diff --git a/dashboard/lib/atlas-don/platform.go b/dashboard/lib/atlas-don/platform.go new file mode 100644 index 00000000000..51cc81eef5e --- /dev/null +++ b/dashboard/lib/atlas-don/platform.go @@ -0,0 +1,49 @@ +package atlas_don + +import "fmt" + +type PlatformOpts struct { + // Platform is infrastructure deployment platform: docker or k8s + Platform string + LabelFilters map[string]string + LabelFilter string + LegendString string + LabelQuery string +} + +// PlatformPanelOpts generate different queries for "docker" and "k8s" deployment platforms +func PlatformPanelOpts(platform string) PlatformOpts { + po := PlatformOpts{ + LabelFilters: map[string]string{ + "instance": `=~"${instance}"`, + "commit": `=~"${commit:pipe}"`, + }, + } + switch platform { + case "kubernetes": + po.LabelFilters = map[string]string{ + // TODO: sometimes I can see my PodMonitor selector, sometimes I don't + // TODO: is it prometheus-operator issue or do we really need "job" selector for k8s? + // TODO: works without it + //"job": `=~"${instance}"`, + "namespace": `=~"${namespace}"`, + "pod": `=~"${pod}"`, + } + po.LabelFilter = "job" + po.LegendString = "pod" + break + case "docker": + po.LabelFilters = map[string]string{ + "instance": `=~"${instance}"`, + } + po.LabelFilter = "instance" + po.LegendString = "instance" + break + default: + panic(fmt.Sprintf("failed to generate Platform dependent queries, unknown platform: %s", platform)) + } + for key, value := range po.LabelFilters { + po.LabelQuery += key + value + ", " + } + return po +} From 87a0dd3a261d9fc15bf829474db99658d3e1ea4d Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Tue, 19 Mar 2024 11:37:14 +0100 Subject: [PATCH 02/11] feat(dashboard): add panels included env var --- .../chainlink-cluster/dashboard/cmd/deploy.go | 25 +++-- dashboard/lib/atlas-don/component.go | 94 +++++++++++-------- dashboard/lib/atlas-don/platform.go | 3 +- dashboard/lib/config.go | 24 +++-- dashboard/lib/k8s-pods/component.go | 8 ++ 5 files changed, 98 insertions(+), 56 deletions(-) diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index ffb5397b370..74b308be0b7 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -7,6 +7,7 @@ import ( core_don "github.com/smartcontractkit/chainlink/dashboard-lib/lib/core-don" k8spods "github.com/smartcontractkit/chainlink/dashboard-lib/lib/k8s-pods" waspdb "github.com/smartcontractkit/wasp/dashboard" + "strings" ) const ( @@ -39,15 +40,21 @@ func main() { ), ) } - db.Add( - atlas_don.New( - atlas_don.Props{ - PrometheusDataSource: cfg.DataSources.Prometheus, - PlatformOpts: atlas_don.PlatformPanelOpts(cfg.Platform), - OcrVersion: "ocr", - }, - ), - ) + if cfg.PanelsIncluded["ocr"] || cfg.PanelsIncluded["ocr2"] || cfg.PanelsIncluded["ocr3"] { + for key := range cfg.PanelsIncluded { + if strings.Contains(key, "ocr") { + db.Add( + atlas_don.New( + atlas_don.Props{ + PrometheusDataSource: cfg.DataSources.Prometheus, + PlatformOpts: atlas_don.PlatformPanelOpts(cfg.Platform), + OcrVersion: key, + }, + ), + ) + } + } + } // TODO: refactor as a component later // addWASPRows(db, cfg) if err := db.Deploy(); err != nil { diff --git a/dashboard/lib/atlas-don/component.go b/dashboard/lib/atlas-don/component.go index 71ba8e965bb..b2980d542fb 100644 --- a/dashboard/lib/atlas-don/component.go +++ b/dashboard/lib/atlas-don/component.go @@ -7,6 +7,7 @@ import ( "github.com/K-Phoen/grabana/row" "github.com/K-Phoen/grabana/stat" "github.com/K-Phoen/grabana/target/prometheus" + "github.com/K-Phoen/grabana/timeseries" "github.com/K-Phoen/grabana/variable/query" ) @@ -26,6 +27,14 @@ func vars(p Props) []dashboard.Option { query.Request(fmt.Sprintf("label_values(%s)", "contract")), query.Sort(query.NumericalAsc), ), + dashboard.VariableAsQuery( + "feed_id_name", + query.DataSource(p.PrometheusDataSource), + query.Multiple(), + query.IncludeAll(), + query.Request(fmt.Sprintf("label_values(%s)", "feed_id_name")), + query.Sort(query.NumericalAsc), + ), } } @@ -43,7 +52,7 @@ func ocrContractConfigOracle(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(12), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }} - {{oracle}}"), ), ), @@ -54,7 +63,7 @@ func ocrContractConfigOracle(p Props) []dashboard.Option { gauge.DataSource(p.PrometheusDataSource), gauge.Description("set to one as long as an oracle is on a feed"), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }} - {{oracle}}"), ), ), @@ -75,7 +84,7 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -85,7 +94,7 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -98,7 +107,7 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -108,7 +117,7 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -121,7 +130,7 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -131,7 +140,7 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -152,7 +161,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -162,7 +171,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -175,7 +184,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -185,7 +194,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -198,7 +207,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -208,7 +217,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -221,7 +230,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -231,7 +240,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -244,7 +253,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -254,7 +263,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -267,7 +276,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -277,7 +286,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -290,7 +299,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.ValueFontSize(20), stat.Span(6), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -300,7 +309,7 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { gauge.Orientation(gauge.OrientationVertical), gauge.DataSource(p.PrometheusDataSource), gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`contract=~"${contract}"}`, + ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), ), ), @@ -308,26 +317,30 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { } } -func ocrTelemetryFeed(p Props) []dashboard.Option { - return []dashboard.Option{ - dashboard.Row("OCR Telemetry Feed", - row.Collapse(), - ), - } -} - -func ocrTelemetryP2P(p Props) []dashboard.Option { +func roundEpochProgression(p Props) []dashboard.Option { return []dashboard.Option{ - dashboard.Row("OCR Telemetry P2P", - row.Collapse(), - ), - } -} - -func ocrTelemetryOthers(p Props) []dashboard.Option { - return []dashboard.Option{ - dashboard.Row("OCR Telemetry Others", + dashboard.Row("Round / Epoch Progression", row.Collapse(), + row.WithTimeSeries( + "Agreed Epoch Progression", + timeseries.Span(4), + timeseries.Height("200px"), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{`+p.PlatformOpts.LabelQuery+`}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{network_name}} - {{contract}}"), + ), + ), + row.WithTimeSeries( + "Round Epoch Progression", + timeseries.Span(4), + timeseries.Height("200px"), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_epoch_round{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}`, + prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{network_name}} - {{contract}}"), + ), + ), ), } } @@ -337,5 +350,6 @@ func New(p Props) []dashboard.Option { opts = append(opts, ocrContractConfigOracle(p)...) opts = append(opts, ocrContractConfigNodes(p)...) opts = append(opts, ocrContractConfigDelta(p)...) + opts = append(opts, roundEpochProgression(p)...) return opts } diff --git a/dashboard/lib/atlas-don/platform.go b/dashboard/lib/atlas-don/platform.go index 51cc81eef5e..1f2e2e05b89 100644 --- a/dashboard/lib/atlas-don/platform.go +++ b/dashboard/lib/atlas-don/platform.go @@ -17,6 +17,7 @@ func PlatformPanelOpts(platform string) PlatformOpts { LabelFilters: map[string]string{ "instance": `=~"${instance}"`, "commit": `=~"${commit:pipe}"`, + "contract": `=~"${contract}"`, }, } switch platform { @@ -25,7 +26,7 @@ func PlatformPanelOpts(platform string) PlatformOpts { // TODO: sometimes I can see my PodMonitor selector, sometimes I don't // TODO: is it prometheus-operator issue or do we really need "job" selector for k8s? // TODO: works without it - //"job": `=~"${instance}"`, + "job": `=~"${job}"`, "namespace": `=~"${namespace}"`, "pod": `=~"${pod}"`, } diff --git a/dashboard/lib/config.go b/dashboard/lib/config.go index eae6e956fc7..c4c12f6f67d 100644 --- a/dashboard/lib/config.go +++ b/dashboard/lib/config.go @@ -1,13 +1,17 @@ package dashboardlib -import "os" +import ( + "os" + "strings" +) type EnvConfig struct { - Platform string - GrafanaURL string - GrafanaToken string - GrafanaFolder string - DataSources DataSources + Platform string + GrafanaURL string + GrafanaToken string + GrafanaFolder string + DataSources DataSources + PanelsIncluded map[string]bool } type DataSources struct { @@ -57,6 +61,13 @@ func ReadEnvDeployOpts() EnvConfig { if prometheusDataSourceName == "" { L.Fatal().Msg("PROMETHEUS_DATA_SOURCE_NAME must be provided") } + panelsIncludedString := os.Getenv("PANELS_INCLUDED") + panelsIncludedArray := strings.Split(panelsIncludedString, ",") + panelsIncluded := make(map[string]bool) + for _, panelName := range panelsIncludedArray { + panelsIncluded[panelName] = true + } + return EnvConfig{ GrafanaURL: grafanaURL, GrafanaToken: grafanaToken, @@ -66,5 +77,6 @@ func ReadEnvDeployOpts() EnvConfig { Loki: loki, Prometheus: prom, }, + PanelsIncluded: panelsIncluded, } } diff --git a/dashboard/lib/k8s-pods/component.go b/dashboard/lib/k8s-pods/component.go index 4ef90c3012f..1ea83294f42 100644 --- a/dashboard/lib/k8s-pods/component.go +++ b/dashboard/lib/k8s-pods/component.go @@ -34,6 +34,14 @@ func vars(p Props) []dashboard.Option { query.Request("label_values(kube_pod_container_info{namespace=\"$namespace\"}, pod)"), query.Sort(query.NumericalAsc), ), + dashboard.VariableAsQuery( + "job", + query.DataSource(p.PrometheusDataSource), + query.Multiple(), + query.IncludeAll(), + query.Request("label_values(up{namespace=\"$namespace\"}, job)"), + query.Sort(query.NumericalAsc), + ), } } From 8df2da0534f989ee78110a1aacbc0642b57c7d36 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Thu, 21 Mar 2024 13:33:10 +0100 Subject: [PATCH 03/11] feat(dashboard): add panels for round epoch progression --- dashboard-lib/atlas-don/component.go | 44 ++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/dashboard-lib/atlas-don/component.go b/dashboard-lib/atlas-don/component.go index b2980d542fb..9db9b19e526 100644 --- a/dashboard-lib/atlas-don/component.go +++ b/dashboard-lib/atlas-don/component.go @@ -8,6 +8,7 @@ import ( "github.com/K-Phoen/grabana/stat" "github.com/K-Phoen/grabana/target/prometheus" "github.com/K-Phoen/grabana/timeseries" + "github.com/K-Phoen/grabana/timeseries/axis" "github.com/K-Phoen/grabana/variable/query" ) @@ -324,21 +325,54 @@ func roundEpochProgression(p Props) []dashboard.Option { row.WithTimeSeries( "Agreed Epoch Progression", timeseries.Span(4), - timeseries.Height("200px"), + timeseries.Height("300px"), timeseries.DataSource(p.PrometheusDataSource), + timeseries.Axis( + axis.Unit("short"), + ), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{network_name}} - {{contract}}"), + ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}`, + prometheus.Legend("{{feed_id_name}}"), ), ), row.WithTimeSeries( "Round Epoch Progression", timeseries.Span(4), - timeseries.Height("200px"), + timeseries.Height("300px"), timeseries.DataSource(p.PrometheusDataSource), + timeseries.Axis( + axis.Unit("short"), + ), timeseries.WithPrometheusTarget( ``+p.OcrVersion+`_telemetry_epoch_round{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{network_name}} - {{contract}}"), + prometheus.Legend("{{oracle}}"), + ), + ), + row.WithTimeSeries( + "Rounds Started", + timeseries.Description("Tracks individual nodes firing \"new round\" message via telemetry (not part of P2P messages)"), + timeseries.Span(4), + timeseries.Height("300px"), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.Axis( + axis.Unit("short"), + ), + timeseries.WithPrometheusTarget( + `rate(`+p.OcrVersion+`_telemetry_round_started_total{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}[1m])`, + prometheus.Legend("{{oracle}}"), + ), + ), + row.WithTimeSeries( + "Telemetry Ingested", + timeseries.Span(12), + timeseries.Height("300px"), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.Axis( + axis.Unit("short"), + ), + timeseries.WithPrometheusTarget( + `rate(`+p.OcrVersion+`_telemetry_ingested_total{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}[1m])`, + prometheus.Legend("{{oracle}}"), ), ), ), From f3958503527a813de23320dce39bf1cd64606296 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Wed, 27 Mar 2024 12:14:02 +0100 Subject: [PATCH 04/11] feat(dashboard): add summary don panels --- dashboard-lib/atlas-don/component.go | 41 ++++++++++++++++++++++++++++ dashboard-lib/atlas-don/utils.go | 5 ++++ dashboard-lib/k8s-pods/component.go | 8 ++++++ 3 files changed, 54 insertions(+) create mode 100644 dashboard-lib/atlas-don/utils.go diff --git a/dashboard-lib/atlas-don/component.go b/dashboard-lib/atlas-don/component.go index 9db9b19e526..0bdec7c5258 100644 --- a/dashboard-lib/atlas-don/component.go +++ b/dashboard-lib/atlas-don/component.go @@ -39,6 +39,46 @@ func vars(p Props) []dashboard.Option { } } +func summary(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("Summary", + row.Collapse(), + row.WithStat( + "Telemetry Down", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextValueAndName), + stat.Description("Which jobs are not receiving any telemetry?"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(12), + stat.Span(12), + stat.WithPrometheusTarget( + `bool:`+p.OcrVersion+`_telemetry_down{`+p.PlatformOpts.LabelQuery+`} == 1`, + prometheus.Legend("{{job}} | {{report_type}}"), + ), + ), + row.WithStat( + "Oracles Down", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextName), + stat.Description("Which NOPs are not providing any telemetry?"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(12), + stat.Span(12), + stat.ValueType(stat.Last), + stat.WithPrometheusTarget( + `bool:`+p.OcrVersion+`_oracle_telemetry_down_except_telemetry_down{job=~"${job}", oracle!="csa_unknown"} == 1`, + prometheus.Legend("{{oracle}} | {{report_type}}"), + ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#FF0000", Value: float64Ptr(1.0)}, + }), + ), + ), + } +} + func ocrContractConfigOracle(p Props) []dashboard.Option { return []dashboard.Option{ dashboard.Row("OCR Contract Oracle", @@ -381,6 +421,7 @@ func roundEpochProgression(p Props) []dashboard.Option { func New(p Props) []dashboard.Option { opts := vars(p) + opts = append(opts, summary(p)...) opts = append(opts, ocrContractConfigOracle(p)...) opts = append(opts, ocrContractConfigNodes(p)...) opts = append(opts, ocrContractConfigDelta(p)...) diff --git a/dashboard-lib/atlas-don/utils.go b/dashboard-lib/atlas-don/utils.go new file mode 100644 index 00000000000..03221905493 --- /dev/null +++ b/dashboard-lib/atlas-don/utils.go @@ -0,0 +1,5 @@ +package atlas_don + +func float64Ptr(input float64) *float64 { + return &input +} diff --git a/dashboard-lib/k8s-pods/component.go b/dashboard-lib/k8s-pods/component.go index 1ea83294f42..7203fde6574 100644 --- a/dashboard-lib/k8s-pods/component.go +++ b/dashboard-lib/k8s-pods/component.go @@ -42,6 +42,14 @@ func vars(p Props) []dashboard.Option { query.Request("label_values(up{namespace=\"$namespace\"}, job)"), query.Sort(query.NumericalAsc), ), + dashboard.VariableAsQuery( + "service", + query.DataSource(p.PrometheusDataSource), + query.Multiple(), + query.IncludeAll(), + query.Request("label_values(up{namespace=\"$namespace\"}, service)"), + query.Sort(query.NumericalAsc), + ), } } From 0cfb8f970324c34e0959cd4f5ef16ccc9d7d156a Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Thu, 11 Apr 2024 13:52:21 +0200 Subject: [PATCH 05/11] feat(dashboard): cleanup --- charts/chainlink-cluster/README.md | 5 +- .../chainlink-cluster/dashboard/cmd/deploy.go | 30 +- dashboard-lib/atlas-don/component.go | 415 ++++++++++-------- dashboard-lib/atlas-don/platform.go | 20 +- dashboard-lib/k8s-pods/component.go | 12 +- 5 files changed, 272 insertions(+), 210 deletions(-) diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 0fbbd5d16df..a0a4b8a78cc 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -151,6 +151,9 @@ We are using [Grabana](https://github.com/K-Phoen/grabana) lib to create dashboa You can also select dashboard platform in `INFRA_PLATFORM` either `kubernetes` or `docker` +You can select the dashboard panels with `PANELS_INCLUDED` which is a list of panel names separated by comma +If you don't specify it will include core panels by default + ``` export LOKI_TENANT_ID=promtail export LOKI_URL=... @@ -159,7 +162,7 @@ export GRAFANA_TOKEN=... export PROMETHEUS_DATA_SOURCE_NAME=Thanos export LOKI_DATA_SOURCE_NAME=Loki export INFRA_PLATFORM=kubernetes -export GRAFANA_FOLDER=CRIB +export GRAFANA_FOLDER=DashboardCoreDebug export DASHBOARD_NAME=CL-Cluster devspace run dashboard_deploy diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index 7881aecd162..5c8e37b7ad4 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -18,23 +18,17 @@ func main() { dashboard.Tags([]string{"generated"}), }, ) - db.Add( - core_don.New( - core_don.Props{ - PrometheusDataSource: cfg.DataSources.Prometheus, - PlatformOpts: core_don.PlatformPanelOpts(cfg.Platform), - }, - ), - ) - if cfg.Platform == "kubernetes" { + if len(cfg.PanelsIncluded) == 1 || cfg.PanelsIncluded["core"] { db.Add( - k8spods.New( - k8spods.Props{ + core_don.New( + core_don.Props{ PrometheusDataSource: cfg.DataSources.Prometheus, - LokiDataSource: cfg.DataSources.Loki, + PlatformOpts: core_don.PlatformPanelOpts(cfg.Platform), }, ), ) + // TODO: refactor as a component later + addWASPRows(db, cfg) } if cfg.PanelsIncluded["ocr"] || cfg.PanelsIncluded["ocr2"] || cfg.PanelsIncluded["ocr3"] { for key := range cfg.PanelsIncluded { @@ -51,8 +45,16 @@ func main() { } } } - // TODO: refactor as a component later - addWASPRows(db, cfg) + if cfg.Platform == "kubernetes" { + db.Add( + k8spods.New( + k8spods.Props{ + PrometheusDataSource: cfg.DataSources.Prometheus, + LokiDataSource: cfg.DataSources.Loki, + }, + ), + ) + } if err := db.Deploy(); err != nil { lib.L.Fatal().Err(err).Msg("failed to deploy the dashboard") } diff --git a/dashboard-lib/atlas-don/component.go b/dashboard-lib/atlas-don/component.go index 0bdec7c5258..7b0e920fd03 100644 --- a/dashboard-lib/atlas-don/component.go +++ b/dashboard-lib/atlas-don/component.go @@ -3,7 +3,6 @@ package atlas_don import ( "fmt" "github.com/K-Phoen/grabana/dashboard" - "github.com/K-Phoen/grabana/gauge" "github.com/K-Phoen/grabana/row" "github.com/K-Phoen/grabana/stat" "github.com/K-Phoen/grabana/target/prometheus" @@ -25,7 +24,7 @@ func vars(p Props) []dashboard.Option { query.DataSource(p.PrometheusDataSource), query.Multiple(), query.IncludeAll(), - query.Request(fmt.Sprintf("label_values(%s)", "contract")), + query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job"}, %s)`, "contract")), query.Sort(query.NumericalAsc), ), dashboard.VariableAsQuery( @@ -33,7 +32,7 @@ func vars(p Props) []dashboard.Option { query.DataSource(p.PrometheusDataSource), query.Multiple(), query.IncludeAll(), - query.Request(fmt.Sprintf("label_values(%s)", "feed_id_name")), + query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job", contract="$contract"}, %s)`, "feed_id_name")), query.Sort(query.NumericalAsc), ), } @@ -42,20 +41,23 @@ func vars(p Props) []dashboard.Option { func summary(p Props) []dashboard.Option { return []dashboard.Option{ dashboard.Row("Summary", - row.Collapse(), row.WithStat( "Telemetry Down", stat.DataSource(p.PrometheusDataSource), - stat.Text(stat.TextValueAndName), + stat.Text(stat.TextName), stat.Description("Which jobs are not receiving any telemetry?"), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), stat.ValueFontSize(12), - stat.Span(12), + stat.Span(4), stat.WithPrometheusTarget( `bool:`+p.OcrVersion+`_telemetry_down{`+p.PlatformOpts.LabelQuery+`} == 1`, prometheus.Legend("{{job}} | {{report_type}}"), ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#008000", Value: float64Ptr(0.0)}, + {Color: "#FF0000", Value: float64Ptr(1.0)}, + }), ), row.WithStat( "Oracles Down", @@ -65,13 +67,90 @@ func summary(p Props) []dashboard.Option { stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), stat.ValueFontSize(12), - stat.Span(12), + stat.Span(4), stat.ValueType(stat.Last), stat.WithPrometheusTarget( `bool:`+p.OcrVersion+`_oracle_telemetry_down_except_telemetry_down{job=~"${job}", oracle!="csa_unknown"} == 1`, prometheus.Legend("{{oracle}} | {{report_type}}"), ), stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#008000", Value: float64Ptr(0.0)}, + {Color: "#FF0000", Value: float64Ptr(1.0)}, + }), + ), + row.WithStat( + "Feeds reporting failure", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextName), + stat.Description("Which feeds are failing to report?"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(12), + stat.Span(4), + stat.ValueType(stat.Last), + stat.WithPrometheusTarget( + `bool:`+p.OcrVersion+`_feed_reporting_failure_except_feed_telemetry_down{job=~"${job}", oracle!="csa_unknown"} == 1`, + prometheus.Legend("{{feed_id_name}} on {{job}}"), + ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#008000", Value: float64Ptr(0.0)}, + {Color: "#FF0000", Value: float64Ptr(1.0)}, + }), + ), + row.WithStat( + "Feed telemetry Down", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextName), + stat.Description("Which feeds are not receiving any telemetry?"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(12), + stat.Span(4), + stat.ValueType(stat.Last), + stat.WithPrometheusTarget( + `bool:`+p.OcrVersion+`_feed_telemetry_down_except_telemetry_down{job=~"${job}"} == 1`, + prometheus.Legend("{{feed_id_name}} on {{job}}"), + ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#008000", Value: float64Ptr(0.0)}, + {Color: "#FF0000", Value: float64Ptr(1.0)}, + }), + ), + row.WithStat( + "Oracles no observations", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextName), + stat.Description("Which NOPs are not providing observations?"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(12), + stat.Span(4), + stat.ValueType(stat.Last), + stat.WithPrometheusTarget( + `bool:`+p.OcrVersion+`_oracle_blind_except_telemetry_down{job=~"${job}"} == 1`, + prometheus.Legend("{{oracle}} | {{report_type}}"), + ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#008000", Value: float64Ptr(0.0)}, + {Color: "#FF0000", Value: float64Ptr(1.0)}, + }), + ), + row.WithStat( + "Oracles not contributing observations to feeds", + stat.DataSource(p.PrometheusDataSource), + stat.Text(stat.TextName), + stat.Description("Which oracles are failing to make observations on feeds they should be participating in?"), + stat.Orientation(stat.OrientationHorizontal), + stat.TitleFontSize(12), + stat.ValueFontSize(12), + stat.Span(4), + stat.ValueType(stat.Last), + stat.WithPrometheusTarget( + `bool:`+p.OcrVersion+`_oracle_feed_no_observations_except_oracle_blind_except_feed_reporting_failure_except_feed_telemetry_down{job=~"${job}"} == 1`, + prometheus.Legend("{{oracle}} | {{report_type}}"), + ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#008000", Value: float64Ptr(0.0)}, {Color: "#FF0000", Value: float64Ptr(1.0)}, }), ), @@ -94,95 +173,150 @@ func ocrContractConfigOracle(p Props) []dashboard.Option { stat.Span(12), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }} - {{oracle}}"), + prometheus.Legend("{{ contract }} - {{oracle}}"), ), + stat.AbsoluteThresholds([]stat.ThresholdStep{ + {Color: "#FF0000", Value: float64Ptr(0.0)}, + {Color: "#008000", Value: float64Ptr(1.0)}, + }), ), - row.WithGauge( - "OCR Contract Oracle Active", - gauge.Span(12), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.Description("set to one as long as an oracle is on a feed"), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }} - {{oracle}}"), + ), + } +} + +func ocrContractConfigNodes(p Props) []dashboard.Option { + return []dashboard.Option{ + dashboard.Row("DON Nodes", + row.Collapse(), + row.WithTimeSeries( + "Number of observations from MessageObserve sent", + timeseries.Span(12), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.Legend(timeseries.ToTheRight), + timeseries.Axis( + axis.Min(0), + ), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_observe_total_nop_count{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{feed_id_name}}"), + ), + timeseries.WithPrometheusTarget( + `avg(2 * `+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`} + 4)`, + prometheus.Legend("Max nodes"), + ), + timeseries.WithPrometheusTarget( + `avg(2 * `+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`} + 1)`, + prometheus.Legend("Min nodes"), ), ), ), } } -func ocrContractConfigNodes(p Props) []dashboard.Option { +func priceReporting(p Props) []dashboard.Option { return []dashboard.Option{ - dashboard.Row("OCR Contract Config Nodes", + dashboard.Row("Price Reporting", row.Collapse(), - row.WithStat( - "Node Count", - stat.DataSource(p.PrometheusDataSource), - stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationHorizontal), - stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), - stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + row.WithTimeSeries( + "Ask observation in MessageObserve sent", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), ), ), - row.WithGauge( - "Node Count", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + row.WithTimeSeries( + "Price observation in MessageObserve sent", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), ), ), - row.WithStat( - "Max Faulty Node Count", - stat.DataSource(p.PrometheusDataSource), - stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationHorizontal), - stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), - stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + row.WithTimeSeries( + "Bid observation in MessageObserve sent", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), ), ), - row.WithGauge( - "Max Faulty Node Count", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + row.WithTimeSeries( + "Ask MessagePropose observations", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_propose_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), ), ), - row.WithStat( - "Max Round Node Count", - stat.DataSource(p.PrometheusDataSource), - stat.Text(stat.TextValueAndName), - stat.Orientation(stat.OrientationHorizontal), - stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), - stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + row.WithTimeSeries( + "Price MessagePropose observations", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_propose_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ), + row.WithTimeSeries( + "Bid MessagePropose observations", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_propose_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ), + row.WithTimeSeries( + "Total number of observations included in MessagePropose", + timeseries.Span(12), + timeseries.Description("How often is a node's observation included in the report?"), + timeseries.Legend(timeseries.ToTheRight), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `rate(`+p.OcrVersion+`_telemetry_message_propose_observation_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, + prometheus.Legend("{{oracle}}"), + ), + ), + row.WithTimeSeries( + "Total MessageObserve sent", + timeseries.Span(12), + timeseries.Description("From an individual node's perspective, how often are they sending an observation?"), + timeseries.Legend(timeseries.ToTheRight), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `rate(`+p.OcrVersion+`_telemetry_message_observe_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, + prometheus.Legend("{{oracle}}"), ), ), - row.WithGauge( - "Max Round Node Count", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + row.WithTimeSeries( + "P2P messages received", + timeseries.Span(12), + timeseries.Height("600px"), + timeseries.Description("From an individual node's perspective, how many messages are they receiving from other nodes? Uses ocr_telemetry_p2p_received_total"), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `sum by (sender, receiver) (increase(`+p.OcrVersion+`_telemetry_p2p_received_total{job=~"${job}"}"}[5m]))`, + prometheus.Legend("{{sender}} > {{receiver}}"), ), ), ), @@ -199,21 +333,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "relativeDeviationThreshold", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_alpha{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), row.WithStat( @@ -222,21 +347,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "maxContractValueAgeSeconds", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_c_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), row.WithStat( @@ -245,21 +361,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "observationGracePeriodSeconds", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_grace_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), row.WithStat( @@ -268,21 +375,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "badEpochTimeoutSeconds", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_progress_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), row.WithStat( @@ -291,21 +389,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "resendIntervalSeconds", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_resend_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), row.WithStat( @@ -314,21 +403,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "roundIntervalSeconds", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_round_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), row.WithStat( @@ -337,21 +417,12 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { stat.Text(stat.TextValueAndName), stat.Orientation(stat.OrientationHorizontal), stat.TitleFontSize(12), - stat.ValueFontSize(20), - stat.Span(6), + stat.ValueFontSize(28), + stat.Span(4), + stat.SparkLine(), stat.WithPrometheusTarget( ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), - ), - ), - row.WithGauge( - "transmissionStageTimeoutSeconds", - gauge.Span(6), - gauge.Orientation(gauge.OrientationVertical), - gauge.DataSource(p.PrometheusDataSource), - gauge.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_config_delta_stage_seconds{`+p.PlatformOpts.LabelQuery+`}`, - prometheus.Legend("{{"+p.PlatformOpts.LegendString+"}} - {{ contract }}"), + prometheus.Legend("{{ contract }}"), ), ), ), @@ -371,7 +442,7 @@ func roundEpochProgression(p Props) []dashboard.Option { axis.Unit("short"), ), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}`, + ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{feed_id_name=~"${feed_id_name}"}`, prometheus.Legend("{{feed_id_name}}"), ), ), @@ -384,7 +455,7 @@ func roundEpochProgression(p Props) []dashboard.Option { axis.Unit("short"), ), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_epoch_round{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}`, + ``+p.OcrVersion+`_telemetry_epoch_round{feed_id_name=~"${feed_id_name}"}`, prometheus.Legend("{{oracle}}"), ), ), @@ -398,7 +469,7 @@ func roundEpochProgression(p Props) []dashboard.Option { axis.Unit("short"), ), timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_round_started_total{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}[1m])`, + `rate(`+p.OcrVersion+`_telemetry_round_started_total{feed_id_name=~"${feed_id_name}"}[1m])`, prometheus.Legend("{{oracle}}"), ), ), @@ -410,8 +481,9 @@ func roundEpochProgression(p Props) []dashboard.Option { timeseries.Axis( axis.Unit("short"), ), + timeseries.Legend(timeseries.ToTheRight), timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_ingested_total{`+p.PlatformOpts.LabelQuery+`feed_id_name=~"${feed_id_name}"}[1m])`, + `rate(`+p.OcrVersion+`_telemetry_ingested_total{feed_id_name=~"${feed_id_name}"}[1m])`, prometheus.Legend("{{oracle}}"), ), ), @@ -424,6 +496,7 @@ func New(p Props) []dashboard.Option { opts = append(opts, summary(p)...) opts = append(opts, ocrContractConfigOracle(p)...) opts = append(opts, ocrContractConfigNodes(p)...) + opts = append(opts, priceReporting(p)...) opts = append(opts, ocrContractConfigDelta(p)...) opts = append(opts, roundEpochProgression(p)...) return opts diff --git a/dashboard-lib/atlas-don/platform.go b/dashboard-lib/atlas-don/platform.go index 1f2e2e05b89..ab8443e2111 100644 --- a/dashboard-lib/atlas-don/platform.go +++ b/dashboard-lib/atlas-don/platform.go @@ -15,28 +15,20 @@ type PlatformOpts struct { func PlatformPanelOpts(platform string) PlatformOpts { po := PlatformOpts{ LabelFilters: map[string]string{ - "instance": `=~"${instance}"`, - "commit": `=~"${commit:pipe}"`, - "contract": `=~"${contract}"`, + "contract": `=~"${contract}"`, + "feed_id_name": `=~"${feed_id_name}"`, }, } switch platform { case "kubernetes": - po.LabelFilters = map[string]string{ - // TODO: sometimes I can see my PodMonitor selector, sometimes I don't - // TODO: is it prometheus-operator issue or do we really need "job" selector for k8s? - // TODO: works without it - "job": `=~"${job}"`, - "namespace": `=~"${namespace}"`, - "pod": `=~"${pod}"`, - } + po.LabelFilters["namespace"] = `=~"${namespace}"` + po.LabelFilters["job"] = `=~"${job}"` + po.LabelFilters["pod"] = `=~"${pod}"` po.LabelFilter = "job" po.LegendString = "pod" break case "docker": - po.LabelFilters = map[string]string{ - "instance": `=~"${instance}"`, - } + po.LabelFilters["instance"] = `=~"${instance}"` po.LabelFilter = "instance" po.LegendString = "instance" break diff --git a/dashboard-lib/k8s-pods/component.go b/dashboard-lib/k8s-pods/component.go index 7203fde6574..fc66cdb001b 100644 --- a/dashboard-lib/k8s-pods/component.go +++ b/dashboard-lib/k8s-pods/component.go @@ -26,14 +26,6 @@ func vars(p Props) []dashboard.Option { query.Request("label_values(namespace)"), query.Sort(query.NumericalAsc), ), - dashboard.VariableAsQuery( - "pod", - query.DataSource(p.PrometheusDataSource), - query.Multiple(), - query.IncludeAll(), - query.Request("label_values(kube_pod_container_info{namespace=\"$namespace\"}, pod)"), - query.Sort(query.NumericalAsc), - ), dashboard.VariableAsQuery( "job", query.DataSource(p.PrometheusDataSource), @@ -43,11 +35,11 @@ func vars(p Props) []dashboard.Option { query.Sort(query.NumericalAsc), ), dashboard.VariableAsQuery( - "service", + "pod", query.DataSource(p.PrometheusDataSource), query.Multiple(), query.IncludeAll(), - query.Request("label_values(up{namespace=\"$namespace\"}, service)"), + query.Request("label_values(up{namespace=\"$namespace\", job=\"$job\"}, pod)"), query.Sort(query.NumericalAsc), ), } From 0f6cde13426f3a41daa730dd21feecdeb9d20762 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Thu, 11 Apr 2024 15:19:21 +0200 Subject: [PATCH 06/11] chore: modify sonarkube code duplication exclusion --- sonar-project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index 616d7883e19..b0e773b1048 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -58,8 +58,8 @@ sonar.cpd.exclusions=\ **/integration-tests/contracts/ethereum_keeper_contracts.go,\ integration-tests/contracts/ethereum_contracts_seth.go,\ integration-tests/contracts/ethereum_contracts_seth.go,\ -integration-tests/actions/seth/actions.go -dashboard/** +integration-tests/actions/seth/actions.go,\ +dashboard-lib/** # Tests' root folder, inclusions (tests to check and count) and exclusions sonar.tests=. From 0ffba95e6e781510b3735e7ba1c4fc3eff398b41 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Fri, 12 Apr 2024 12:49:29 +0200 Subject: [PATCH 07/11] feat(dashboard): delete dashboard --- .../chainlink-cluster/dashboard/cmd/delete.go | 19 ++++++++++++ .../chainlink-cluster/dashboard/cmd/deploy.go | 2 +- dashboard-lib/config.go | 7 +++-- dashboard-lib/dashboard.go | 31 +++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 charts/chainlink-cluster/dashboard/cmd/delete.go diff --git a/charts/chainlink-cluster/dashboard/cmd/delete.go b/charts/chainlink-cluster/dashboard/cmd/delete.go new file mode 100644 index 00000000000..45b4b11d67f --- /dev/null +++ b/charts/chainlink-cluster/dashboard/cmd/delete.go @@ -0,0 +1,19 @@ +package main + +import ( + lib "github.com/smartcontractkit/chainlink/dashboard-lib" +) + +func main() { + cfg := lib.ReadEnvDeployOpts() + db := lib.NewDashboard(cfg.Name, cfg, nil) + err := db.Delete() + if err != nil { + lib.L.Fatal().Err(err).Msg("failed to delete the dashboard") + } + lib.L.Info(). + Str("Name", db.Name). + Str("GrafanaURL", db.DeployOpts.GrafanaURL). + Str("GrafanaFolder", db.DeployOpts.GrafanaFolder). + Msg("Dashboard deleted") +} diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index 5c8e37b7ad4..09037e43dce 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -18,7 +18,7 @@ func main() { dashboard.Tags([]string{"generated"}), }, ) - if len(cfg.PanelsIncluded) == 1 || cfg.PanelsIncluded["core"] { + if len(cfg.PanelsIncluded) == 0 || cfg.PanelsIncluded["core"] { db.Add( core_don.New( core_don.Props{ diff --git a/dashboard-lib/config.go b/dashboard-lib/config.go index a5417994f56..2e0b9cad993 100644 --- a/dashboard-lib/config.go +++ b/dashboard-lib/config.go @@ -65,8 +65,11 @@ func ReadEnvDeployOpts() EnvConfig { panelsIncludedString := os.Getenv("PANELS_INCLUDED") panelsIncludedArray := strings.Split(panelsIncludedString, ",") panelsIncluded := make(map[string]bool) - for _, panelName := range panelsIncludedArray { - panelsIncluded[panelName] = true + + if panelsIncludedString != "" { + for _, panelName := range panelsIncludedArray { + panelsIncluded[panelName] = true + } } ba := os.Getenv("GRAFANA_BASIC_AUTH") diff --git a/dashboard-lib/dashboard.go b/dashboard-lib/dashboard.go index 33371a6163a..70892586bb0 100644 --- a/dashboard-lib/dashboard.go +++ b/dashboard-lib/dashboard.go @@ -3,6 +3,7 @@ package dashboard_lib import ( "context" "encoding/json" + "fmt" "github.com/K-Phoen/grabana" "github.com/K-Phoen/grabana/dashboard" "github.com/pkg/errors" @@ -72,6 +73,36 @@ func (m *Dashboard) AddSDKPanel(panel map[string]interface{}) { m.SDKPanels = append(m.SDKPanels, panel) } +func (m *Dashboard) Delete() error { + ctx := context.Background() + var client *grabana.Client + if m.DeployOpts.GrafanaBasicAuthUser != "" && m.DeployOpts.GrafanaBasicAuthPassword != "" { + L.Info().Msg("Authorizing using BasicAuth") + client = grabana.NewClient( + &http.Client{}, + m.DeployOpts.GrafanaURL, + grabana.WithBasicAuth(m.DeployOpts.GrafanaBasicAuthUser, m.DeployOpts.GrafanaBasicAuthPassword), + ) + } else { + L.Info().Msg("Authorizing using Bearer token") + client = grabana.NewClient( + &http.Client{}, + m.DeployOpts.GrafanaURL, + grabana.WithAPIToken(m.DeployOpts.GrafanaToken), + ) + } + db, err := client.GetDashboardByTitle(ctx, m.Name) + if err != nil { + return errors.Wrap(err, "failed to get the dashboard") + } + fmt.Println(db.UID) + errDelete := client.DeleteDashboard(ctx, db.UID) + if errDelete != nil { + return errors.Wrap(errDelete, "failed to delete the dashboard") + } + return nil +} + func (m *Dashboard) build() (dashboard.Builder, error) { b, err := dashboard.New( m.Name, From 108920f9d9936b61fc3f970e7b9a2fd80b40a6a9 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Fri, 12 Apr 2024 17:24:36 +0200 Subject: [PATCH 08/11] feat(dashboard): adapt for multiple ocr versions --- .../chainlink-cluster/dashboard/cmd/deploy.go | 2 +- dashboard-lib/atlas-don/component.go | 248 ++++++++++-------- dashboard-lib/atlas-don/platform.go | 8 +- 3 files changed, 148 insertions(+), 110 deletions(-) diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index 09037e43dce..24c3af4589b 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -37,7 +37,7 @@ func main() { atlas_don.New( atlas_don.Props{ PrometheusDataSource: cfg.DataSources.Prometheus, - PlatformOpts: atlas_don.PlatformPanelOpts(cfg.Platform), + PlatformOpts: atlas_don.PlatformPanelOpts(cfg.Platform, key), OcrVersion: key, }, ), diff --git a/dashboard-lib/atlas-don/component.go b/dashboard-lib/atlas-don/component.go index 7b0e920fd03..31099f99505 100644 --- a/dashboard-lib/atlas-don/component.go +++ b/dashboard-lib/atlas-don/component.go @@ -18,6 +18,11 @@ type Props struct { } func vars(p Props) []dashboard.Option { + variableFeedId := "feed_id_name" + if p.OcrVersion == "ocr2" { + variableFeedId = "feed_id" + } + return []dashboard.Option{ dashboard.VariableAsQuery( "contract", @@ -28,11 +33,11 @@ func vars(p Props) []dashboard.Option { query.Sort(query.NumericalAsc), ), dashboard.VariableAsQuery( - "feed_id_name", + variableFeedId, query.DataSource(p.PrometheusDataSource), query.Multiple(), query.IncludeAll(), - query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job", contract="$contract"}, %s)`, "feed_id_name")), + query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job", contract="$contract"}, %s)`, variableFeedId)), query.Sort(query.NumericalAsc), ), } @@ -214,112 +219,141 @@ func ocrContractConfigNodes(p Props) []dashboard.Option { } func priceReporting(p Props) []dashboard.Option { - return []dashboard.Option{ - dashboard.Row("Price Reporting", - row.Collapse(), - row.WithTimeSeries( - "Ask observation in MessageObserve sent", - timeseries.Span(12), - timeseries.Legend(timeseries.ToTheRight), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Price observation in MessageObserve sent", - timeseries.Span(12), - timeseries.Legend(timeseries.ToTheRight), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Bid observation in MessageObserve sent", - timeseries.Span(12), - timeseries.Legend(timeseries.ToTheRight), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Ask MessagePropose observations", - timeseries.Span(12), - timeseries.Legend(timeseries.ToTheRight), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_propose_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Price MessagePropose observations", - timeseries.Span(12), - timeseries.Legend(timeseries.ToTheRight), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_propose_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Bid MessagePropose observations", - timeseries.Span(12), - timeseries.Legend(timeseries.ToTheRight), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_propose_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Total number of observations included in MessagePropose", - timeseries.Span(12), - timeseries.Description("How often is a node's observation included in the report?"), - timeseries.Legend(timeseries.ToTheRight), - timeseries.Axis( - axis.Min(0), - ), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_message_propose_observation_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "Total MessageObserve sent", - timeseries.Span(12), - timeseries.Description("From an individual node's perspective, how often are they sending an observation?"), - timeseries.Legend(timeseries.ToTheRight), - timeseries.Axis( - axis.Min(0), - ), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_message_observe_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, - prometheus.Legend("{{oracle}}"), - ), - ), - row.WithTimeSeries( - "P2P messages received", - timeseries.Span(12), - timeseries.Height("600px"), - timeseries.Description("From an individual node's perspective, how many messages are they receiving from other nodes? Uses ocr_telemetry_p2p_received_total"), - timeseries.Axis( - axis.Min(0), - ), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - `sum by (sender, receiver) (increase(`+p.OcrVersion+`_telemetry_p2p_received_total{job=~"${job}"}"}[5m]))`, - prometheus.Legend("{{sender}} > {{receiver}}"), - ), - ), + telemetryObservationAsk := row.WithTimeSeries( + "Ask observation in MessageObserve sent", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryObservation := row.WithTimeSeries( + "Price observation in MessageObserve sent", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryObservationBid := row.WithTimeSeries( + "Bid observation in MessageObserve sent", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryMessageProposeObservationAsk := row.WithTimeSeries( + "Ask MessagePropose observations", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_propose_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryMessageProposeObservation := row.WithTimeSeries( + "Price MessagePropose observations", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_propose_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryMessageProposeObservationBid := row.WithTimeSeries( + "Bid MessagePropose observations", + timeseries.Span(12), + timeseries.Legend(timeseries.ToTheRight), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_propose_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryMessageProposeObservationTotal := row.WithTimeSeries( + "Total number of observations included in MessagePropose", + timeseries.Span(12), + timeseries.Description("How often is a node's observation included in the report?"), + timeseries.Legend(timeseries.ToTheRight), + timeseries.Axis( + axis.Min(0), ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `rate(`+p.OcrVersion+`_telemetry_message_propose_observation_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryMessageObserveTotal := row.WithTimeSeries( + "Total MessageObserve sent", + timeseries.Span(12), + timeseries.Description("From an individual node's perspective, how often are they sending an observation?"), + timeseries.Legend(timeseries.ToTheRight), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `rate(`+p.OcrVersion+`_telemetry_message_observe_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, + prometheus.Legend("{{oracle}}"), + ), + ) + + telemetryP2PReceivedTotal := row.WithTimeSeries( + "P2P messages received", + timeseries.Span(12), + timeseries.Height("600px"), + timeseries.Description("From an individual node's perspective, how many messages are they receiving from other nodes? Uses ocr_telemetry_p2p_received_total"), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `sum by (sender, receiver) (increase(`+p.OcrVersion+`_telemetry_p2p_received_total{job=~"${job}"}[5m]))`, + prometheus.Legend("{{sender}} > {{receiver}}"), + ), + ) + + panels := []row.Option{ + row.Collapse(), + } + + switch p.OcrVersion { + case "ocr": + break + case "ocr2": + break + case "ocr3": + panels = append(panels, telemetryObservationAsk) + panels = append(panels, telemetryObservation) + panels = append(panels, telemetryObservationBid) + panels = append(panels, telemetryMessageProposeObservationAsk) + panels = append(panels, telemetryMessageProposeObservation) + panels = append(panels, telemetryMessageProposeObservationBid) + panels = append(panels, telemetryMessageProposeObservationTotal) + panels = append(panels, telemetryMessageObserveTotal) + panels = append(panels, telemetryP2PReceivedTotal) + break + } + + return []dashboard.Option{ + dashboard.Row("Price Reporting", panels...), } } diff --git a/dashboard-lib/atlas-don/platform.go b/dashboard-lib/atlas-don/platform.go index ab8443e2111..6ff9716c111 100644 --- a/dashboard-lib/atlas-don/platform.go +++ b/dashboard-lib/atlas-don/platform.go @@ -12,11 +12,15 @@ type PlatformOpts struct { } // PlatformPanelOpts generate different queries for "docker" and "k8s" deployment platforms -func PlatformPanelOpts(platform string) PlatformOpts { +func PlatformPanelOpts(platform string, ocrVersion string) PlatformOpts { + variableFeedId := "feed_id_name" + if ocrVersion == "ocr2" { + variableFeedId = "feed_id" + } po := PlatformOpts{ LabelFilters: map[string]string{ "contract": `=~"${contract}"`, - "feed_id_name": `=~"${feed_id_name}"`, + variableFeedId: `=~"${` + variableFeedId + `}"`, }, } switch platform { From 6fb5cf58dbd6c32f23377dd84394cc29e3fcdeda Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Tue, 16 Apr 2024 15:02:17 +0200 Subject: [PATCH 09/11] feat(dashboard): DON dashboard for all ocr versions --- .../chainlink-cluster/dashboard/cmd/deploy.go | 12 + dashboard-lib/atlas-don/component.go | 220 ++++++++++++------ dashboard-lib/atlas-don/platform.go | 23 +- 3 files changed, 176 insertions(+), 79 deletions(-) diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index 24c3af4589b..171cf523ef5 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -5,6 +5,7 @@ import ( lib "github.com/smartcontractkit/chainlink/dashboard-lib" atlas_don "github.com/smartcontractkit/chainlink/dashboard-lib/atlas-don" core_don "github.com/smartcontractkit/chainlink/dashboard-lib/core-don" + core_node_components "github.com/smartcontractkit/chainlink/dashboard-lib/core-node-components" k8spods "github.com/smartcontractkit/chainlink/dashboard-lib/k8s-pods" waspdb "github.com/smartcontractkit/wasp/dashboard" "strings" @@ -30,6 +31,17 @@ func main() { // TODO: refactor as a component later addWASPRows(db, cfg) } + if cfg.PanelsIncluded["components"] { + db.Add( + core_node_components.New( + core_node_components.Props{ + PrometheusDataSource: cfg.DataSources.Prometheus, + PlatformOpts: core_node_components.PlatformPanelOpts(cfg.Platform), + }, + ), + ) + } + if cfg.PanelsIncluded["ocr"] || cfg.PanelsIncluded["ocr2"] || cfg.PanelsIncluded["ocr3"] { for key := range cfg.PanelsIncluded { if strings.Contains(key, "ocr") { diff --git a/dashboard-lib/atlas-don/component.go b/dashboard-lib/atlas-don/component.go index 31099f99505..39218c7aea8 100644 --- a/dashboard-lib/atlas-don/component.go +++ b/dashboard-lib/atlas-don/component.go @@ -18,29 +18,45 @@ type Props struct { } func vars(p Props) []dashboard.Option { - variableFeedId := "feed_id_name" - if p.OcrVersion == "ocr2" { - variableFeedId = "feed_id" + variableFeedId := "feed_id" + if p.OcrVersion == "ocr3" { + variableFeedId = "feed_id_name" } - return []dashboard.Option{ - dashboard.VariableAsQuery( - "contract", - query.DataSource(p.PrometheusDataSource), - query.Multiple(), - query.IncludeAll(), - query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job"}, %s)`, "contract")), - query.Sort(query.NumericalAsc), - ), - dashboard.VariableAsQuery( - variableFeedId, - query.DataSource(p.PrometheusDataSource), - query.Multiple(), - query.IncludeAll(), - query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job", contract="$contract"}, %s)`, variableFeedId)), - query.Sort(query.NumericalAsc), - ), + variableQueryContract := dashboard.VariableAsQuery( + "contract", + query.DataSource(p.PrometheusDataSource), + query.Multiple(), + query.IncludeAll(), + query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job"}, %s)`, "contract")), + query.Sort(query.NumericalAsc), + ) + + variableQueryFeedId := dashboard.VariableAsQuery( + variableFeedId, + query.DataSource(p.PrometheusDataSource), + query.Multiple(), + query.IncludeAll(), + query.Request(fmt.Sprintf(`label_values(`+p.OcrVersion+`_contract_config_f{job="$job", contract="$contract"}, %s)`, variableFeedId)), + query.Sort(query.NumericalAsc), + ) + + variables := []dashboard.Option{ + variableQueryContract, + } + + switch p.OcrVersion { + case "ocr": + break + case "ocr2": + variables = append(variables, variableQueryFeedId) + break + case "ocr3": + variables = append(variables, variableQueryFeedId) + break } + + return variables } func summary(p Props) []dashboard.Option { @@ -170,14 +186,13 @@ func ocrContractConfigOracle(p Props) []dashboard.Option { row.WithStat( "OCR Contract Oracle Active", stat.DataSource(p.PrometheusDataSource), - stat.Text(stat.TextValueAndName), + stat.Text(stat.TextName), stat.Description("set to one as long as an oracle is on a feed"), stat.Orientation(stat.OrientationHorizontal), - stat.TitleFontSize(12), - stat.ValueFontSize(20), + stat.ValueFontSize(12), stat.Span(12), stat.WithPrometheusTarget( - ``+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`}`, + `sum(`+p.OcrVersion+`_contract_oracle_active{`+p.PlatformOpts.LabelQuery+`}) by (contract, oracle)`, prometheus.Legend("{{ contract }} - {{oracle}}"), ), stat.AbsoluteThresholds([]stat.ThresholdStep{ @@ -190,42 +205,102 @@ func ocrContractConfigOracle(p Props) []dashboard.Option { } func ocrContractConfigNodes(p Props) []dashboard.Option { + variableFeedId := "feed_id" + if p.OcrVersion == "ocr3" { + variableFeedId = "feed_id_name" + } + + var options []timeseries.Option + + options = append(options, timeseries.Span(12), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.Legend(timeseries.ToTheRight), + timeseries.Axis( + axis.Min(0), + ), + ) + + switch p.OcrVersion { + case "ocr": + options = append(options, timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`}`, + prometheus.Legend("{{contract}}"), + )) + break + case "ocr2": + options = append(options, timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_contract_config_n{`+p.PlatformOpts.LabelQuery+`}`, + prometheus.Legend("{{"+variableFeedId+"}}"), + )) + break + case "ocr3": + options = append(options, timeseries.WithPrometheusTarget( + ``+p.OcrVersion+`_telemetry_message_observe_total_nop_count{contract=~"${contract}", `+variableFeedId+`=~"${`+variableFeedId+`}", job=~"${job}"}`, + prometheus.Legend("{{"+variableFeedId+"}}"), + )) + break + } + + options = append(options, + timeseries.WithPrometheusTarget( + `avg(2 * `+p.OcrVersion+`_contract_config_r_max{`+p.PlatformOpts.LabelQuery+`} + 4)`, + prometheus.Legend("Max nodes"), + ), + timeseries.WithPrometheusTarget( + `avg(2 * `+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`} + 1)`, + prometheus.Legend("Min nodes"), + ), + ) + return []dashboard.Option{ dashboard.Row("DON Nodes", row.Collapse(), row.WithTimeSeries( - "Number of observations from MessageObserve sent", - timeseries.Span(12), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.Legend(timeseries.ToTheRight), - timeseries.Axis( - axis.Min(0), - ), - timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_observe_total_nop_count{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, - prometheus.Legend("{{feed_id_name}}"), - ), - timeseries.WithPrometheusTarget( - `avg(2 * `+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`} + 4)`, - prometheus.Legend("Max nodes"), - ), - timeseries.WithPrometheusTarget( - `avg(2 * `+p.OcrVersion+`_contract_config_f{`+p.PlatformOpts.LabelQuery+`} + 1)`, - prometheus.Legend("Min nodes"), - ), + "Number of NOPs", + options..., ), ), } } func priceReporting(p Props) []dashboard.Option { + telemetryP2PReceivedTotal := row.WithTimeSeries( + "P2P messages received", + timeseries.Span(12), + timeseries.Height("600px"), + timeseries.Description("From an individual node's perspective, how many messages are they receiving from other nodes? Uses ocr_telemetry_p2p_received_total"), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `sum by (sender, receiver) (increase(`+p.OcrVersion+`_telemetry_p2p_received_total{job=~"${job}"}[5m]))`, + prometheus.Legend("{{sender}} > {{receiver}}"), + ), + ) + + telemetryP2PReceivedTotalRate := row.WithTimeSeries( + "P2P messages received Rate", + timeseries.Span(12), + timeseries.Height("600px"), + timeseries.Description("From an individual node's perspective, how many messages are they receiving from other nodes? Uses ocr_telemetry_p2p_received_total"), + timeseries.Axis( + axis.Min(0), + ), + timeseries.DataSource(p.PrometheusDataSource), + timeseries.WithPrometheusTarget( + `sum by (sender, receiver) (rate(`+p.OcrVersion+`_telemetry_p2p_received_total{job=~"${job}"}[5m]))`, + prometheus.Legend("{{sender}} > {{receiver}}"), + ), + ) + telemetryObservationAsk := row.WithTimeSeries( "Ask observation in MessageObserve sent", timeseries.Span(12), timeseries.Legend(timeseries.ToTheRight), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + ``+p.OcrVersion+`_telemetry_observation_ask{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{oracle}}"), ), ) @@ -236,7 +311,7 @@ func priceReporting(p Props) []dashboard.Option { timeseries.Legend(timeseries.ToTheRight), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + ``+p.OcrVersion+`_telemetry_observation{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{oracle}}"), ), ) @@ -247,7 +322,7 @@ func priceReporting(p Props) []dashboard.Option { timeseries.Legend(timeseries.ToTheRight), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + ``+p.OcrVersion+`_telemetry_observation_bid{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{oracle}}"), ), ) @@ -258,7 +333,7 @@ func priceReporting(p Props) []dashboard.Option { timeseries.Legend(timeseries.ToTheRight), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_propose_observation_ask{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + ``+p.OcrVersion+`_telemetry_message_propose_observation_ask{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{oracle}}"), ), ) @@ -269,7 +344,7 @@ func priceReporting(p Props) []dashboard.Option { timeseries.Legend(timeseries.ToTheRight), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_propose_observation{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + ``+p.OcrVersion+`_telemetry_message_propose_observation{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{oracle}}"), ), ) @@ -280,7 +355,7 @@ func priceReporting(p Props) []dashboard.Option { timeseries.Legend(timeseries.ToTheRight), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_message_propose_observation_bid{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}`, + ``+p.OcrVersion+`_telemetry_message_propose_observation_bid{`+p.PlatformOpts.LabelQuery+`}`, prometheus.Legend("{{oracle}}"), ), ) @@ -295,7 +370,7 @@ func priceReporting(p Props) []dashboard.Option { ), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_message_propose_observation_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, + `rate(`+p.OcrVersion+`_telemetry_message_propose_observation_total{`+p.PlatformOpts.LabelQuery+`}[5m])`, prometheus.Legend("{{oracle}}"), ), ) @@ -310,36 +385,31 @@ func priceReporting(p Props) []dashboard.Option { ), timeseries.DataSource(p.PrometheusDataSource), timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_message_observe_total{contract=~"${contract}", feed_id_name=~"${feed_id_name}", job=~"${job}"}[5m])`, + `rate(`+p.OcrVersion+`_telemetry_message_observe_total{`+p.PlatformOpts.LabelQuery+`}[5m])`, prometheus.Legend("{{oracle}}"), ), ) - telemetryP2PReceivedTotal := row.WithTimeSeries( - "P2P messages received", - timeseries.Span(12), - timeseries.Height("600px"), - timeseries.Description("From an individual node's perspective, how many messages are they receiving from other nodes? Uses ocr_telemetry_p2p_received_total"), - timeseries.Axis( - axis.Min(0), - ), - timeseries.DataSource(p.PrometheusDataSource), - timeseries.WithPrometheusTarget( - `sum by (sender, receiver) (increase(`+p.OcrVersion+`_telemetry_p2p_received_total{job=~"${job}"}[5m]))`, - prometheus.Legend("{{sender}} > {{receiver}}"), - ), - ) - panels := []row.Option{ row.Collapse(), } switch p.OcrVersion { case "ocr": + panels = append(panels, telemetryP2PReceivedTotal) + panels = append(panels, telemetryP2PReceivedTotalRate) + panels = append(panels, telemetryObservation) + panels = append(panels, telemetryMessageObserveTotal) break case "ocr2": + panels = append(panels, telemetryP2PReceivedTotal) + panels = append(panels, telemetryP2PReceivedTotalRate) + panels = append(panels, telemetryObservation) + panels = append(panels, telemetryMessageObserveTotal) break case "ocr3": + panels = append(panels, telemetryP2PReceivedTotal) + panels = append(panels, telemetryP2PReceivedTotalRate) panels = append(panels, telemetryObservationAsk) panels = append(panels, telemetryObservation) panels = append(panels, telemetryObservationBid) @@ -348,7 +418,6 @@ func priceReporting(p Props) []dashboard.Option { panels = append(panels, telemetryMessageProposeObservationBid) panels = append(panels, telemetryMessageProposeObservationTotal) panels = append(panels, telemetryMessageObserveTotal) - panels = append(panels, telemetryP2PReceivedTotal) break } @@ -464,6 +533,11 @@ func ocrContractConfigDelta(p Props) []dashboard.Option { } func roundEpochProgression(p Props) []dashboard.Option { + variableFeedId := "feed_id" + if p.OcrVersion == "ocr3" { + variableFeedId = "feed_id_name" + } + return []dashboard.Option{ dashboard.Row("Round / Epoch Progression", row.Collapse(), @@ -476,8 +550,8 @@ func roundEpochProgression(p Props) []dashboard.Option { axis.Unit("short"), ), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{feed_id_name=~"${feed_id_name}"}`, - prometheus.Legend("{{feed_id_name}}"), + ``+p.OcrVersion+`_telemetry_feed_agreed_epoch{`+variableFeedId+`=~"${`+variableFeedId+`}"}`, + prometheus.Legend("{{"+variableFeedId+"}}"), ), ), row.WithTimeSeries( @@ -489,7 +563,7 @@ func roundEpochProgression(p Props) []dashboard.Option { axis.Unit("short"), ), timeseries.WithPrometheusTarget( - ``+p.OcrVersion+`_telemetry_epoch_round{feed_id_name=~"${feed_id_name}"}`, + ``+p.OcrVersion+`_telemetry_epoch_round{`+variableFeedId+`=~"${`+variableFeedId+`}"}`, prometheus.Legend("{{oracle}}"), ), ), @@ -503,7 +577,7 @@ func roundEpochProgression(p Props) []dashboard.Option { axis.Unit("short"), ), timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_round_started_total{feed_id_name=~"${feed_id_name}"}[1m])`, + `rate(`+p.OcrVersion+`_telemetry_round_started_total{`+variableFeedId+`=~"${`+variableFeedId+`}"}[1m])`, prometheus.Legend("{{oracle}}"), ), ), @@ -517,7 +591,7 @@ func roundEpochProgression(p Props) []dashboard.Option { ), timeseries.Legend(timeseries.ToTheRight), timeseries.WithPrometheusTarget( - `rate(`+p.OcrVersion+`_telemetry_ingested_total{feed_id_name=~"${feed_id_name}"}[1m])`, + `rate(`+p.OcrVersion+`_telemetry_ingested_total{`+variableFeedId+`=~"${`+variableFeedId+`}"}[1m])`, prometheus.Legend("{{oracle}}"), ), ), @@ -531,7 +605,7 @@ func New(p Props) []dashboard.Option { opts = append(opts, ocrContractConfigOracle(p)...) opts = append(opts, ocrContractConfigNodes(p)...) opts = append(opts, priceReporting(p)...) - opts = append(opts, ocrContractConfigDelta(p)...) opts = append(opts, roundEpochProgression(p)...) + opts = append(opts, ocrContractConfigDelta(p)...) return opts } diff --git a/dashboard-lib/atlas-don/platform.go b/dashboard-lib/atlas-don/platform.go index 6ff9716c111..b55bab3f7ef 100644 --- a/dashboard-lib/atlas-don/platform.go +++ b/dashboard-lib/atlas-don/platform.go @@ -13,16 +13,27 @@ type PlatformOpts struct { // PlatformPanelOpts generate different queries for "docker" and "k8s" deployment platforms func PlatformPanelOpts(platform string, ocrVersion string) PlatformOpts { - variableFeedId := "feed_id_name" - if ocrVersion == "ocr2" { - variableFeedId = "feed_id" - } po := PlatformOpts{ LabelFilters: map[string]string{ - "contract": `=~"${contract}"`, - variableFeedId: `=~"${` + variableFeedId + `}"`, + "contract": `=~"${contract}"`, }, } + + variableFeedId := "feed_id" + if ocrVersion == "ocr3" { + variableFeedId = "feed_id_name" + } + + switch ocrVersion { + case "ocr": + break + case "ocr2": + po.LabelFilters[variableFeedId] = `=~"${` + variableFeedId + `}"` + break + case "ocr3": + po.LabelFilters[variableFeedId] = `=~"${` + variableFeedId + `}"` + break + } switch platform { case "kubernetes": po.LabelFilters["namespace"] = `=~"${namespace}"` From b0d2b63e77f3f92b1b29de6f1a8c2489c34fba45 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Tue, 16 Apr 2024 15:07:53 +0200 Subject: [PATCH 10/11] feat(dashbord): separated PR for core components --- charts/chainlink-cluster/dashboard/cmd/deploy.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/charts/chainlink-cluster/dashboard/cmd/deploy.go b/charts/chainlink-cluster/dashboard/cmd/deploy.go index 171cf523ef5..24c3af4589b 100644 --- a/charts/chainlink-cluster/dashboard/cmd/deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/deploy.go @@ -5,7 +5,6 @@ import ( lib "github.com/smartcontractkit/chainlink/dashboard-lib" atlas_don "github.com/smartcontractkit/chainlink/dashboard-lib/atlas-don" core_don "github.com/smartcontractkit/chainlink/dashboard-lib/core-don" - core_node_components "github.com/smartcontractkit/chainlink/dashboard-lib/core-node-components" k8spods "github.com/smartcontractkit/chainlink/dashboard-lib/k8s-pods" waspdb "github.com/smartcontractkit/wasp/dashboard" "strings" @@ -31,17 +30,6 @@ func main() { // TODO: refactor as a component later addWASPRows(db, cfg) } - if cfg.PanelsIncluded["components"] { - db.Add( - core_node_components.New( - core_node_components.Props{ - PrometheusDataSource: cfg.DataSources.Prometheus, - PlatformOpts: core_node_components.PlatformPanelOpts(cfg.Platform), - }, - ), - ) - } - if cfg.PanelsIncluded["ocr"] || cfg.PanelsIncluded["ocr2"] || cfg.PanelsIncluded["ocr3"] { for key := range cfg.PanelsIncluded { if strings.Contains(key, "ocr") { From 4217618a938d344b76cf407da73d713fb9c46db8 Mon Sep 17 00:00:00 2001 From: Clement Erena Date: Tue, 16 Apr 2024 16:52:54 +0200 Subject: [PATCH 11/11] feat(dashboard): correct escaping for k8s vars --- dashboard-lib/k8s-pods/component.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard-lib/k8s-pods/component.go b/dashboard-lib/k8s-pods/component.go index fc66cdb001b..df9a6ac6a69 100644 --- a/dashboard-lib/k8s-pods/component.go +++ b/dashboard-lib/k8s-pods/component.go @@ -31,7 +31,7 @@ func vars(p Props) []dashboard.Option { query.DataSource(p.PrometheusDataSource), query.Multiple(), query.IncludeAll(), - query.Request("label_values(up{namespace=\"$namespace\"}, job)"), + query.Request(`label_values(up{namespace="$namespace"}, job)`), query.Sort(query.NumericalAsc), ), dashboard.VariableAsQuery( @@ -39,7 +39,7 @@ func vars(p Props) []dashboard.Option { query.DataSource(p.PrometheusDataSource), query.Multiple(), query.IncludeAll(), - query.Request("label_values(up{namespace=\"$namespace\", job=\"$job\"}, pod)"), + query.Request(`label_values(up{namespace="$namespace", job="$job"}, pod)`), query.Sort(query.NumericalAsc), ), }