Skip to content

Commit

Permalink
fix: vm metrics and grafana dashboard
Browse files Browse the repository at this point in the history
  • Loading branch information
@gabheadz
gabheadz committed Dec 26, 2024
1 parent c8dfdf1 commit 990e902
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 19 deletions.
60 changes: 45 additions & 15 deletions ...les/grafana/ADF Sender-1734921285644.json → ...les/grafana/ADF Sender-1735038654302.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Live counter of routed messages",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -155,6 +156,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Live counter of current channel processes.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -234,6 +236,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Rate of messages delivered/non-delivered via the channel socket.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -316,14 +319,31 @@
"expr": "sum(rate(elixir_adf_message_delivered_count[5m]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"legendFormat": "delivered",
"range": true,
"refId": "A",
"useBackend": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
}
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum(rate(elixir_adf_message_nodelivered_count[5m]))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "non_delivered",
"range": true,
"refId": "B",
"useBackend": false
}
],
"title": "Message Delivery by Sec",
Expand All @@ -334,6 +354,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Time series aggregate of the number of live channel processes.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -423,7 +444,7 @@
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "channels",
"legendFormat": "num of channels",
"range": true,
"refId": "A",
"useBackend": false,
Expand All @@ -441,6 +462,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests per second to the rest and socket endpoints",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -550,14 +572,15 @@
"useBackend": false
}
],
"title": "Rest requests by Sec.",
"title": "Requests by Sec.",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Average response time of rest endpoints",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -637,9 +660,9 @@
{
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(request_path) (elixir_http_request_duration_milliseconds_duration)",
"expr": "sum by(request_path) (rate(elixir_http_request_duration_milliseconds_duration[5m]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"includeNullMetadata": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
Expand All @@ -650,7 +673,7 @@
}
}
],
"title": "Rest requests avg duration (msec)",
"title": "Rest requests avg duration (millis)",
"type": "timeseries"
},
{
Expand Down Expand Up @@ -898,6 +921,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This represents the total memory in bytes used by the Erlang/Elixir Virtual Machine (VM). It is a high-level aggregate metric that combines all types of memory consumption in the VM.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -983,7 +1007,7 @@
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "builder",
"editorMode": "code",
"expr": "elixir_vm_memory_total",
"fullMetaSearch": false,
"includeNullMetadata": true,
Expand All @@ -994,14 +1018,15 @@
"useBackend": false
}
],
"title": "VM Memory",
"title": "VM Memory (bytes)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This metric represents the total memory in bytes used by all processes running in the Erlang/Elixir VM. This includes memory allocated for the process heaps, stacks, and internal data structures required for process execution.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1084,7 +1109,7 @@
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "builder",
"editorMode": "code",
"expr": "elixir_vm_memory_processes",
"fullMetaSearch": false,
"includeNullMetadata": true,
Expand All @@ -1095,14 +1120,15 @@
"useBackend": false
}
],
"title": "VM Process Memory",
"title": "VM Process Memory (bytes)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This metric is the total length of all run queues across all schedulers in the Erlang VM at a given point in time. \nIt includes the sum of CPU-bound work (Processes waiting to execute) and \nIO-bound work (Processes waiting for IO operations to complete)",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1199,6 +1225,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This metric represents the total number of processes in the run queue across all schedulers in the Erlang VM that are specifically waiting for CPU time.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1295,6 +1322,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This metric represents the total length of the IO-bound run queues across all schedulers in the Erlang VM. This metric tracks processes waiting for IO-related operations to complete rather than being ready for immediate CPU execution.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1391,6 +1419,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This metric represents the total amount of memory in bytes used by ETS (Erlang Term Storage) tables in the Erlang/Elixir VM. ETS is a high-performance in-memory storage system built into the Erlang VM, commonly used for storing large amounts of data in key-value format.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1484,14 +1513,15 @@
"useBackend": false
}
],
"title": "VM ETS Memory",
"title": "VM ETS Memory (bytes)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "This metric represents the amount of memory in bytes being used by binaries in the Erlang/Elixir VM. Binaries are: String manipulation (when encoded as binaries), Network communication (e.g., TCP/UDP payloads), File I/O and storage, Efficient handling of large data structures.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1574,7 +1604,7 @@
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "builder",
"editorMode": "code",
"expr": "elixir_vm_memory_binary",
"fullMetaSearch": false,
"includeNullMetadata": true,
Expand All @@ -1585,11 +1615,11 @@
"useBackend": false
}
],
"title": "VM Binary Memory",
"title": "VM Binary Memory (bytes)",
"type": "timeseries"
}
],
"refresh": "auto",
"refresh": "5s",
"schemaVersion": 40,
"tags": [],
"templating": {
Expand All @@ -1603,6 +1633,6 @@
"timezone": "browser",
"title": "ADF Sender",
"uid": "de7ltonluxse8e",
"version": 23,
"version": 28,
"weekStart": ""
}
8 changes: 4 additions & 4 deletions channel-sender/lib/channel_sender_ex/utils/custom_telemetry.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,10 @@ defmodule ChannelSenderEx.Utils.CustomTelemetry do
last_value("elixir.vm.memory.processes", unit: {:byte, :kilobyte}, tags: [:service]),
last_value("elixir.vm.memory.binary", unit: {:byte, :kilobyte}, tags: [:service]),
last_value("elixir.vm.memory.ets", unit: {:byte, :kilobyte}, tags: [:service]),
sum("elixir.vm.total_run_queue_lengths.total", tags: [:service]),
sum("elixir.vm.total_run_queue_lengths.cpu", tags: [:service]),
sum("elixir.vm.total_run_queue_lengths.io", tags: [:service]),
sum("elixir.vm.system_counts_process.count", tags: [:service])
last_value("elixir.vm.total_run_queue_lengths.total", tags: [:service]),
last_value("elixir.vm.total_run_queue_lengths.cpu", tags: [:service]),
last_value("elixir.vm.total_run_queue_lengths.io", tags: [:service]),
last_value("elixir.vm.system_counts_process.count", tags: [:service])
]
end

Expand Down

0 comments on commit 990e902

Please sign in to comment.