From c32daead61caadb35cbf2c15cf111a79b8d8a398 Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Thu, 17 Oct 2024 19:02:24 +0100 Subject: [PATCH] [nyarlathotep] Switch from promscale to VictoriaMetrics Promscale was discontinued early last year, and around the beginning of this year developed the irritating problem of frequently dropping data, which led to me looking at the dashboard less and less often. I tried using the prometheus backfilling tool, but couldn't get it to work - it apparently loaded everything without error, but the metrics just weren't there. It also took 10m+ to load my historic hledger data, which is far worse than the few seconds promscale took. Today I found VictoriaMetrics, which falls into the same sweet spot as promscale: it has an API that supports deleting timeseries and loading data with arbitrary timestamps, and supports the prometheus query language. It works really well so far, is faster for big queries than promscale was, and I've got my dashboard working again! --- hosts/nyarlathotep/configuration.nix | 40 ++----- hosts/nyarlathotep/dashboards/finance.json | 111 ++++++++++-------- ...y => hledger-export-to-victoriametrics.py} | 22 +++- 3 files changed, 90 insertions(+), 83 deletions(-) rename hosts/nyarlathotep/jobs/{hledger-export-to-promscale.py => hledger-export-to-victoriametrics.py} (94%) diff --git a/hosts/nyarlathotep/configuration.nix b/hosts/nyarlathotep/configuration.nix index a2c5d037..1f23dd71 100644 --- a/hosts/nyarlathotep/configuration.nix +++ b/hosts/nyarlathotep/configuration.nix @@ -22,7 +22,6 @@ with lib; let shares = [ "anime" "manga" "misc" "music" "movies" "tv" "torrents" ]; - promscalePort = 9201; prometheusAwairExporterPort = 9517; httpdir = "${toString config.nixfiles.eraseYourDarlings.persistDir}/srv/http"; @@ -359,8 +358,8 @@ in provision = { datasources.settings.datasources = [ { - name = "promscale"; - url = "http://localhost:${toString promscalePort}"; + name = "victoriametrics"; + url = "http://${config.services.victoriametrics.listenAddress}"; type = "prometheus"; } ]; @@ -462,46 +461,27 @@ in }; }; - systemd.services.hledger-export-to-promscale = { - description = "Export personal finance data to promscale"; + systemd.services.hledger-export-to-victoriametrics = { + description = "Export personal finance data to VictoriaMetrics"; startAt = "daily"; path = with pkgs; [ hledger ]; serviceConfig = { ExecStart = let python = pkgs.python3.withPackages (ps: [ ps.requests ]); - in "${python}/bin/python3 ${pkgs.writeText "hledger-export-to-promscale.py" (fileContents ./jobs/hledger-export-to-promscale.py)}"; + in "${python}/bin/python3 ${pkgs.writeText "hledger-export-to-victoriametrics.py" (fileContents ./jobs/hledger-export-to-victoriametrics.py)}"; User = "barrucadu"; Group = "users"; }; environment = { LEDGER_FILE = "/home/barrucadu/s/ledger/combined.journal"; - PROMSCALE_URI = "http://localhost:${toString promscalePort}"; + VICTORIAMETRICS_URI = "http://${config.services.victoriametrics.listenAddress}"; }; }; - nixfiles.oci-containers.pods.promscale = { - containers = { - web = { - image = "timescale/promscale:latest"; - cmd = [ - "-db.host=promscale-db" - "-db.name=postgres" - "-db.password=promscale" - "-db.ssl-mode=allow" - "-web.enable-admin-api=true" - "-metrics.promql.lookback-delta=168h" - ]; - dependsOn = [ "promscale-db" ]; - ports = [{ host = promscalePort; inner = 9201; }]; - }; - db = { - image = "timescaledev/promscale-extension:latest-ts2-pg14"; - environment = { - POSTGRES_PASSWORD = "promscale"; - }; - volumes = [{ name = "pgdata"; inner = "/var/lib/postgresql/data"; }]; - }; - }; + services.victoriametrics = { + enable = true; + listenAddress = "127.0.0.1:8428"; + retentionPeriod = 1200; }; diff --git a/hosts/nyarlathotep/dashboards/finance.json b/hosts/nyarlathotep/dashboards/finance.json index 296f3245..97420886 100644 --- a/hosts/nyarlathotep/dashboards/finance.json +++ b/hosts/nyarlathotep/dashboards/finance.json @@ -76,10 +76,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -149,11 +150,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -225,10 +227,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -298,10 +301,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -368,11 +372,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -441,11 +446,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -514,7 +520,6 @@ "y": 4 }, "id": 21, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", @@ -528,11 +533,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -979,7 +985,6 @@ "y": 4 }, "id": 69, - "links": [], "options": { "legend": { "calcs": [], @@ -1100,11 +1105,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -1227,7 +1233,6 @@ "y": 6 }, "id": 17, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", @@ -1241,11 +1246,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -1312,7 +1318,6 @@ "y": 6 }, "id": 18, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", @@ -1326,11 +1331,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -1397,7 +1403,6 @@ "y": 9 }, "id": 19, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", @@ -1411,11 +1416,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -1482,7 +1488,6 @@ "y": 9 }, "id": 20, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", @@ -1496,11 +1501,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -1890,7 +1896,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "bars", + "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "hideFrom": { @@ -1969,7 +1975,7 @@ }, { "id": "custom.lineInterpolation", - "value": "linear" + "value": "stepAfter" }, { "id": "custom.axisSoftMin", @@ -2055,7 +2061,6 @@ } ], "title": "Cash Flow", - "transformations": [], "transparent": true, "type": "timeseries" }, @@ -2113,7 +2118,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2187,7 +2192,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2261,7 +2266,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2335,7 +2340,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2409,7 +2414,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2483,7 +2488,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2558,7 +2563,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2632,7 +2637,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2706,7 +2711,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2780,7 +2785,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2855,7 +2860,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2928,7 +2933,7 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.2.6", + "pluginVersion": "10.4.10", "targets": [ { "datasource": { @@ -2971,6 +2976,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2984,6 +2990,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineStyle": { "fill": "solid" @@ -3009,7 +3016,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3037,10 +3045,9 @@ "h": 8, "w": 8, "x": 0, - "y": 55 + "y": 39 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -3087,6 +3094,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3100,6 +3108,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineStyle": { "fill": "solid" @@ -3125,7 +3134,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3153,10 +3163,9 @@ "h": 8, "w": 8, "x": 8, - "y": 55 + "y": 39 }, "id": 82, - "links": [], "options": { "legend": { "calcs": [ @@ -3203,6 +3212,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3216,6 +3226,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineStyle": { "fill": "solid" @@ -3241,7 +3252,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3269,10 +3281,9 @@ "h": 8, "w": 8, "x": 16, - "y": 55 + "y": 39 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -3319,6 +3330,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3332,6 +3344,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineStyle": { "fill": "solid" @@ -3357,7 +3370,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3385,10 +3399,9 @@ "h": 8, "w": 8, "x": 0, - "y": 63 + "y": 47 }, "id": 86, - "links": [], "options": { "legend": { "calcs": [ @@ -3444,9 +3457,9 @@ }, { "current": { - "selected": false, - "text": "promscale", - "value": "P3354215654B729CB" + "selected": true, + "text": "victoriametrics", + "value": "PABDA7AB1AD2A1489" }, "hide": 0, "includeAll": false, @@ -3462,7 +3475,7 @@ }, { "current": { - "selected": false, + "selected": true, "text": "GBP", "value": "GBP" }, diff --git a/hosts/nyarlathotep/jobs/hledger-export-to-promscale.py b/hosts/nyarlathotep/jobs/hledger-export-to-victoriametrics.py similarity index 94% rename from hosts/nyarlathotep/jobs/hledger-export-to-promscale.py rename to hosts/nyarlathotep/jobs/hledger-export-to-victoriametrics.py index c0786cf7..1b6713c2 100644 --- a/hosts/nyarlathotep/jobs/hledger-export-to-promscale.py +++ b/hosts/nyarlathotep/jobs/hledger-export-to-victoriametrics.py @@ -16,7 +16,7 @@ if not DRY_RUN: import requests - PROMSCALE_URI = os.environ["PROMSCALE_URI"] + VICTORIAMETRICS_URI = os.environ["VICTORIAMETRICS_URI"] DOB = datetime.datetime(1991, 9, 9) @@ -349,16 +349,30 @@ def metric_quantified_self_age(credits_debits): for name, values in metrics.items(): if not DRY_RUN: - requests.post(f"{PROMSCALE_URI}/delete_series?match[]={name}") + requests.post( + f"{VICTORIAMETRICS_URI}/api/v1/admin/tsdb/delete_series?match[]={name}" + ).raise_for_status() for labels_tuples, samples in values.items(): print(f"Uploading {name} {labels_tuples} ({len(samples)} samples)") labels = dict(labels_tuples) labels["__name__"] = name - json = {"labels": labels, "samples": convert_samples(samples)} + samples = convert_samples(samples) + json = { + "metric": labels, + "values": [v for _, v in samples], + "timestamps": [t for t, _ in samples], + } if DRY_RUN: print(json) else: - requests.post(f"{PROMSCALE_URI}/write", json=json).raise_for_status() + requests.post( + f"{VICTORIAMETRICS_URI}/api/v1/import", json=json + ).raise_for_status() + +if not DRY_RUN: + requests.get( + f"{VICTORIAMETRICS_URI}/internal/resetRollupResultCache" + ).raise_for_status()