From a8251bf2c05ae2a9e9250c42a221fcf987f6eaa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=BCleyman=20Vurucu?= Date: Wed, 14 Jun 2023 11:18:44 +0200 Subject: [PATCH] Nomadder 119 (#141) * nomadder-119 Docu * nomadder-119 Blackbox exporter * nomadder-119 Vector static ip * nomadder-119 BUGFIX: is_master_0 evaluation * nomadder-119 version_nats_server 2.9.16 > 2.9.17 version_nats_prometheus_exporter 0.11.0 > 0.24.0 version_vector_agent 0.29.1 > 0.30.0 nomad_version: 1.5.5 > 1.5.6 version_protoc 22.3 > 23.1 version_proto_buf_cli 1.17.0 > 1.19.0 * nomadder-119 alertmanager integration * nomadder-119 blackbox exporter * nomadder-119 custom envoy proxy * nomadder-119 docu * nomadder-119 vector static ip to nats * nomadder-119 disable host volume for docker driver * nomadder-119 docu certgen * nomadder-119 CI/CD test deployment with jenkins and gitlab * nomadder-119 Test deployment with shared alloc folder * nomadder-119 sysctl set over docker driver * nomadder-119 Delete obsolete script * nomadder-119 Comment * nomadder-119 version_protoc 23.1 > 23.2 * nomadder-119 workaround issue https://github.com/suikast42/nomadder/issues/138 * nomadder-119 refactor systemd nomad restart * nomadder-119 fix docu * nomadder-119 Grafana dashboards and alarms * nomadder-119 Grafana dashboards and alarms --------- Co-authored-by: VURU --- .envrc | 3 + .../consul/handlers/generate_consul_certs.yml | 4 +- ansible/setup/dnsmasq/templates/dnsmasq.j2 | 3 +- .../docker/handlers/generate_docker_certs.yml | 22 +- ansible/setup/docker/tasks/install_docker.yml | 2 +- .../handlers/generate_vault_certs.yml | 2 +- .../nomad/handlers/generate_nomad_certs.yml | 42 +- .../nomad/tasks/copy_nomad_config_files.yml | 8 - ansible/setup/nomad/tasks/main.yml | 68 +- .../nomad/tasks/nomad_systemd_install.yml | 63 +- .../cert/{nomad_client.j2 => nomad_agent.j2} | 4 +- .../nomad/templates/cert/nomad_server.j2 | 17 - .../nomad/templates/conf/nomad_client.j2 | 30 +- .../nomad/templates/service/nomad.service.j2 | 12 +- .../service/nomad_kill_pause_containers.sh.j2 | 5 + .../templates/service/nomad_node_drain.sh.j2 | 2 +- .../setup/stack_observability/tasks/main.yml | 7 +- .../Cluster overview-1676113506033.json | 2140 -- .../dashboards/LogsTest-1680298618653.json | 1010 - .../Loki2.0 Global Metrics-1675942743100.json | 3595 --- .../consul-exporter-dashboard_rev2.json | 570 - .../dashboards/nats-jetsteram-dash.json | 1405 - .../dashboards/nats-jetstream_rev2.json | 1238 - .../dashboards/nats-servers_rev1.json | 965 - .../dashboards/node-exporter-full.json | 23190 ---------------- .../dashboards/nomad-cluster_rev1.json | 971 - .../dashboards/nomad-jobs_rev1.json | 486 - .../AllocRestartingPermanantly.json | 1 + .../BlackBockExporterCheck.json | 1 + .../BackBoneServices/BlackBoxChecks.json | 1 + .../BackBoneServices/Consul Service Down.json | 1 + .../BackBoneServices/Consul Up metric.json | 1 + .../Consul healthcheck critical.json | 1 + .../Consul service failing.json | 1 + .../Alarms/BackBoneServices/Up metrics.json | 1 + .../Failed Image pulls last hour.json | 1 + .../High Log Error Rate Last Hour.json | 1 + .../NodeClockNotSynchronising.json | 1 + .../NodeMetrics/NodeClockSkewDetected.json | 1 + .../NodeMetrics/NodeFileDescriptorLimit.json | 1 + .../NodeFileDescriptorLimit90pct.json | 1 + .../NodeFilesystemAlmostOutOfFiles3pct.json | 1 + .../NodeFilesystemAlmostOutOfFiles5pct.json | 1 + .../NodeFilesystemAlmostOutOfSpace10.json | 1 + .../NodeFilesystemAlmostOutOfSpace15.json | 1 + .../NodeFilesystemFilesFillingUp24h.json | 1 + .../NodeFilesystemFilesFillingUp4h.json | 1 + .../NodeFilesystemSpaceFillingUpNext24h.json | 1 + .../NodeFilesystemSpaceFillingUpNext4h.json | 1 + .../NodeHighNumberConntrackEntriesUsed.json | 1 + .../NodeMetrics/NodeNetworkReceiveErrs.json | 1 + .../NodeMetrics/NodeNetworkTransmitErrs.json | 1 + .../Alarms/NodeMetrics/NodeRAIDDegraded.json | 1 + .../NodeMetrics/NodeRAIDDiskFailure.json | 1 + .../NodeTextFileCollectorScrapeError.json | 1 + .../dashboards/Alarms/BlackBoxChecks.json | 1 + .../Alarms/Failed to pull images.json | 1 + .../exported/dashboards/Alarms/LogAlerts.json | 1 + .../dashboards/General/Alerts Overview.json | 1 + .../exported/dashboards/General/LogsTest.json | 1 + .../General/Loki2.0 Global Metrics.json | 1 + .../dashboards/General/NATS JetStream.json | 1 + .../dashboards/General/NATS Servers.json | 1 + .../General/Traefik Realtime Stats.json | 1 + .../dashboards/General/Traefik 2.2.json | 1 + .../exported/dashboards/Ingress/Traefik.json | 1 + .../NodeExporter/Node Exporter Full.json | 1 + .../Node Exporter Nomadder1 -Nodes.json | 1 + ...porter Nomadder1 -USE Method Cluster.json | 1 + ... Exporter Nomadder1 -USE Method Node.json | 1 + .../NodeExporter/Nodes Overview.json | 1 + .../Consul Overview.json | 1 + .../Nomad Cluster overview.json | 1 + .../Nomad cluster.json | 1 + .../Nomad Cluster Components/Nomad jobs.json | 1 + .../exported/dashboards/demo/Home.json | 1 + .../data-sources/AlertmanagerMimir.json | 1 + .../data-sources/Grafana HTTP API.json | 1 + .../exported/data-sources/Loki.json | 1 + .../exported/data-sources/Mimir.json | 1 + .../OPC UA (Unified Architecture).json | 1 + .../exported/data-sources/Tempo.json | 1 + .../templates/grafana_agent/agent.j2 | 43 +- .../templates/mimir/mimir.j2 | 3 + .../templates/observability.j2 | 93 + .../tasks/install_vector.yml | 4 +- .../tasks/main.yml | 4 +- .../templates/config/agent-config.yaml | 65 +- .../templates/config/vector.toml | 10 +- .../service/grafana.agent.service.j2 | 2 +- ansible/setup/vars/common_vars.yml | 15 +- ansible/setup/vars/versions.yml | 23 +- docs/200_stack_core.adoc | 5 +- docs/202_stack_core_consul.adoc | 2 +- docs/205_stack_core_loki.adoc | 5 + docs/205_stack_core_mimir_prometheus.adoc | 59 + docs/902_cheet_sheet_certtifactes.adoc | 18 + docs/902_cheet_sheet_docker.adoc | 6 + docs/902_cheet_sheet_mimir.adoc | 29 + docs/902_cheet_sheet_nats.adoc | 8 + docs/902_cheet_sheet_ubuntu.adoc | 18 + .../templates/mimir => docs}/addToDocu.txt | 0 docs/images/nmoad_links.png | Bin 0 -> 302029 bytes test_deployments/blackbox-exporter.nomad | 107 + test_deployments/cicd_deployment/Jenkinsfile | 105 + .../cicd_deployment/Jenkinsfile_test | 52 + test_deployments/cicd_deployment/README.MD | 127 + .../cicd_deployment/TODO_Jenkins.MD | 13 + .../cicd_deployment/certgen_test.nomad | 53 + test_deployments/cicd_deployment/cicd.volumes | 19 + .../cicd_deployment/cicd_with_volumes.nomad | 664 + .../cicd_deployment/jenkinsbuild/Dockerfile | 9 + .../cicd_deployment/jenkinsbuild/build.sh | 19 + test_deployments/elasticsearch.nomad | 4 +- test_deployments/setup.sh | 406 - test_deployments/shared_volume.nomad | 44 + 116 files changed, 1795 insertions(+), 36158 deletions(-) rename ansible/setup/nomad/templates/cert/{nomad_client.j2 => nomad_agent.j2} (70%) delete mode 100644 ansible/setup/nomad/templates/cert/nomad_server.j2 create mode 100644 ansible/setup/nomad/templates/service/nomad_kill_pause_containers.sh.j2 delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Cluster overview-1676113506033.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/LogsTest-1680298618653.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Loki2.0 Global Metrics-1675942743100.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/consul-exporter-dashboard_rev2.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetsteram-dash.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetstream_rev2.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-servers_rev1.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/node-exporter-full.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-cluster_rev1.json delete mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-jobs_rev1.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/AllocRestartingPermanantly.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBockExporterCheck.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBoxChecks.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Service Down.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Up metric.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul healthcheck critical.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul service failing.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Up metrics.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/Failed Image pulls last hour.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/High Log Error Rate Last Hour.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockNotSynchronising.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockSkewDetected.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit90pct.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles3pct.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles5pct.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace10.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace15.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp24h.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp4h.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext24h.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext4h.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeHighNumberConntrackEntriesUsed.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkReceiveErrs.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkTransmitErrs.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDegraded.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDiskFailure.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeTextFileCollectorScrapeError.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/BlackBoxChecks.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/Failed to pull images.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/LogAlerts.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Alerts Overview.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/LogsTest.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Loki2.0 Global Metrics.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS JetStream.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS Servers.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik Realtime Stats.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik 2.2.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Ingress/Traefik.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Full.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -Nodes.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Cluster.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Node.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Nodes Overview.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Consul Overview.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad Cluster overview.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad cluster.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad jobs.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/demo/Home.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/AlertmanagerMimir.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Grafana HTTP API.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Loki.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Mimir.json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/OPC UA (Unified Architecture).json create mode 100644 ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Tempo.json rename {ansible/setup/stack_observability/templates/mimir => docs}/addToDocu.txt (100%) create mode 100644 docs/images/nmoad_links.png create mode 100644 test_deployments/blackbox-exporter.nomad create mode 100644 test_deployments/cicd_deployment/Jenkinsfile create mode 100644 test_deployments/cicd_deployment/Jenkinsfile_test create mode 100644 test_deployments/cicd_deployment/README.MD create mode 100644 test_deployments/cicd_deployment/TODO_Jenkins.MD create mode 100644 test_deployments/cicd_deployment/certgen_test.nomad create mode 100644 test_deployments/cicd_deployment/cicd.volumes create mode 100644 test_deployments/cicd_deployment/cicd_with_volumes.nomad create mode 100644 test_deployments/cicd_deployment/jenkinsbuild/Dockerfile create mode 100644 test_deployments/cicd_deployment/jenkinsbuild/build.sh delete mode 100644 test_deployments/setup.sh create mode 100644 test_deployments/shared_volume.nomad diff --git a/.envrc b/.envrc index 1926154..c7ba02a 100644 --- a/.envrc +++ b/.envrc @@ -7,3 +7,6 @@ export ANSIBLE_CONFIG="$ENVIRONMENT/../ansible.cfg" export ANSIBLE_INVENTORY="$ENVIRONMENT/inventory/hosts.ini" export ANSIBLE_DEBUG=False export DOCKER_CERT_PATH="$ENVIRONMENT/docker_client" + +export PULL_REGISTRY=registry.cloud.private +export PUSH_REGISTRY=10.21.21.41:5001 diff --git a/ansible/setup/consul/handlers/generate_consul_certs.yml b/ansible/setup/consul/handlers/generate_consul_certs.yml index 83bd51a..43de223 100644 --- a/ansible/setup/consul/handlers/generate_consul_certs.yml +++ b/ansible/setup/consul/handlers/generate_consul_certs.yml @@ -63,7 +63,7 @@ src: ../templates/cert/consul_client.j2 dest: "{{cfssl_working_dir}}/consul_client.json" - - name: "Generate consul server certificate(s) in on master 0" + - name: "Generate consul server certificate(s) on master 0" register: cfsslgen when: is_master_host failed_when: cfsslgen.rc != 0 @@ -77,7 +77,7 @@ args: chdir: "{{cfssl_working_dir}}" - - name: "Generate consul client certificate(s) in on master 0" + - name: "Generate consul client certificate(s) on master 0" register: cfsslgen when: is_worker_host failed_when: cfsslgen.rc != 0 diff --git a/ansible/setup/dnsmasq/templates/dnsmasq.j2 b/ansible/setup/dnsmasq/templates/dnsmasq.j2 index b5f544f..171d644 100644 --- a/ansible/setup/dnsmasq/templates/dnsmasq.j2 +++ b/ansible/setup/dnsmasq/templates/dnsmasq.j2 @@ -1,5 +1,4 @@ -listen-address=127.0.0.1 -listen-address={{ ansible_eth1.ipv4.address }} +listen-address=127.0.0.1, {{ ansible_eth1.ipv4.address }} # Enable forward lookup of the 'consul' domain: server=/consul/127.0.0.1#8600 diff --git a/ansible/setup/docker/handlers/generate_docker_certs.yml b/ansible/setup/docker/handlers/generate_docker_certs.yml index d58eb15..d03ac67 100644 --- a/ansible/setup/docker/handlers/generate_docker_certs.yml +++ b/ansible/setup/docker/handlers/generate_docker_certs.yml @@ -47,7 +47,7 @@ file: path: "{{docker_cert_path}}" state: directory - mode: '0400' + mode: '0755' group: docker @@ -73,7 +73,7 @@ dest: "{{cfssl_working_dir}}/docker-server.json" - name: Template docker-client.json - when: is_build_host +# when: is_build_host delegate_to: "{{masters[0]}}" template: force: yes @@ -81,7 +81,7 @@ dest: "{{cfssl_working_dir}}/docker-client.json" - - name: "Generate docker server certificate(s) in on master 0" + - name: "Generate docker server certificate(s) on master 0" register: cfsslgen failed_when: cfsslgen.rc != 0 delegate_to: "{{masters[0]}}" @@ -94,8 +94,8 @@ args: chdir: "{{cfssl_working_dir}}" - - name: "Generate docker client certificate(s) in on master 0" - when: is_build_host + - name: "Generate docker client certificate(s) on master 0" +# when: is_build_host register: cfsslgen failed_when: cfsslgen.rc != 0 delegate_to: "{{masters[0]}}" @@ -111,10 +111,12 @@ - name: "Copy docker server certs" include_tasks: cert_install/tasks/copy_from_master_0.yml loop: - - { src: '{{cfssl_working_dir}}/docker-server.pem', dest: '{{docker_server_cert}}' ,mode: '0400'} - - { src: '{{cfssl_working_dir}}/docker-server-key.pem', dest: '{{docker_server_cert_key}}',mode: '0400'} + - { src: '{{cfssl_working_dir}}/docker-server.pem', dest: '{{docker_server_cert}}' ,mode: '0644'} + - { src: '{{cfssl_working_dir}}/docker-server-key.pem', dest: '{{docker_server_cert_key}}',mode: '0644'} + - { src: '{{cfssl_working_dir}}/docker-client.pem', dest: '{{docker_client_cert}}' ,mode: '0644'} + - { src: '{{cfssl_working_dir}}/docker-client-key.pem', dest: '{{docker_client_cert_key}}',mode: '0644'} - - name: "Copy docker client certs" + - name: "Copy docker client certs from build host to localhost" when: is_build_host run_once: true include_tasks: cert_install/tasks/copy_from_master_0.yml @@ -138,4 +140,6 @@ - cfssl_working_dir: "{{cert_path_master_0}}/{{host_name}}/docker" - docker_cert_path: "{{base_cert_dir}}/docker" - docker_server_cert: "{{docker_cert_path}}/docker-server.pem" - - docker_server_cert_key: "{{docker_cert_path}}/docker-server-key.pem" \ No newline at end of file + - docker_server_cert_key: "{{docker_cert_path}}/docker-server-key.pem" + - docker_client_cert: "{{docker_cert_path}}/docker-client.pem" + - docker_client_cert_key: "{{docker_cert_path}}/docker-client-key.pem" \ No newline at end of file diff --git a/ansible/setup/docker/tasks/install_docker.yml b/ansible/setup/docker/tasks/install_docker.yml index 2497090..ca4de7c 100644 --- a/ansible/setup/docker/tasks/install_docker.yml +++ b/ansible/setup/docker/tasks/install_docker.yml @@ -114,7 +114,7 @@ become: true when: not dockerInstalled.stat.exists or update_certificates|bool == true block: - - name: install_cert handler nomad + - name: install_cert handler docker include_tasks: handlers/generate_docker_certs.yml diff --git a/ansible/setup/master_vault/handlers/generate_vault_certs.yml b/ansible/setup/master_vault/handlers/generate_vault_certs.yml index a20eeb5..a6c9d76 100644 --- a/ansible/setup/master_vault/handlers/generate_vault_certs.yml +++ b/ansible/setup/master_vault/handlers/generate_vault_certs.yml @@ -63,7 +63,7 @@ dest: "{{cfssl_working_dir}}/vault-server.json" - - name: "Generate vault server certificate(s) in on master 0" + - name: "Generate vault server certificate(s) on master 0" register: cfsslgen failed_when: cfsslgen.rc != 0 delegate_to: "{{masters[0]}}" diff --git a/ansible/setup/nomad/handlers/generate_nomad_certs.yml b/ansible/setup/nomad/handlers/generate_nomad_certs.yml index afc522d..c3d6cd8 100644 --- a/ansible/setup/nomad/handlers/generate_nomad_certs.yml +++ b/ansible/setup/nomad/handlers/generate_nomad_certs.yml @@ -62,12 +62,12 @@ path: "{{cfssl_working_dir}}" - - name: Template nomad_server.json + - name: Template nomad_agent.json delegate_to: "{{masters[0]}}" template: force: yes - src: ../templates/cert/nomad_server.j2 - dest: "{{cfssl_working_dir}}/nomad_server.json" + src: ../templates/cert/nomad_agent.j2 + dest: "{{cfssl_working_dir}}/nomad_agent.json" - name: Template nomad_cli.json delegate_to: "{{masters[0]}}" @@ -76,13 +76,6 @@ src: ../templates/cert/nomad_cli.j2 dest: "{{cfssl_working_dir}}/nomad_cli.json" - - name: Template nomad_client.json - delegate_to: "{{masters[0]}}" - template: - force: yes - src: ../templates/cert/nomad_client.j2 - dest: "{{cfssl_working_dir}}/nomad_client.json" - - name: Template nomad_ingress.j2 delegate_to: "{{masters[0]}}" template: @@ -90,9 +83,8 @@ src: ../templates/cert/nomad_ingress.j2 dest: "{{cfssl_working_dir}}/nomad_ingress.json" - - name: "Generate nomad server certificate(s) in on master 0" + - name: "Generate nomad server certificate(s)" register: cfsslgen - when: is_master_host failed_when: cfsslgen.rc != 0 delegate_to: "{{masters[0]}}" shell: | @@ -100,7 +92,7 @@ -ca {{cert_path_master_0}}/cluster-ca/cluster-ca.pem \ -ca-key {{cert_path_master_0}}/cluster-ca/cluster-ca-key.pem \ -config {{cert_path_master_0}}/cfssl/config.json \ - -profile peer nomad_server.json | cfssljson -bare nomad + -profile peer nomad_agent.json | cfssljson -bare nomad cfssl gencert \ -ca {{cert_path_master_0}}/cluster-ca/cluster-ca.pem \ @@ -111,21 +103,6 @@ args: chdir: "{{cfssl_working_dir}}" - - name: "Generate nomad client certificate(s) in on master 0" - register: cfsslgen - when: is_worker_host - failed_when: cfsslgen.rc != 0 - delegate_to: "{{masters[0]}}" - shell: | - cfssl gencert \ - -ca {{cert_path_master_0}}/cluster-ca/cluster-ca.pem \ - -ca-key {{cert_path_master_0}}/cluster-ca/cluster-ca-key.pem \ - -config {{cert_path_master_0}}/cfssl/config.json \ - -profile peer nomad_client.json | cfssljson -bare nomad - - args: - chdir: "{{cfssl_working_dir}}" - - name: "Generate nomad ingress certificate(s) in on master 0" register: cfsslgen when: is_worker_host @@ -155,6 +132,8 @@ loop: - { src: '{{cfssl_working_dir}}/nomad.pem', dest: '{{nomad_cert}}' ,mode: '0644' } - { src: '{{cfssl_working_dir}}/nomad-key.pem', dest: '{{nomad_cert_key}}',mode: '0644' } + - { src: '{{cfssl_working_dir}}/nomad-cli.pem', dest: '{{nomad_cert_cli}}' ,mode: '0644'} + - { src: '{{cfssl_working_dir}}/nomad-cli-key.pem', dest: '{{nomad_cert_cli_key}}',mode: '0644'} - { src: '{{cfssl_working_dir}}/nomad_ingress.pem', dest: '{{nomad_cert_ingress}}' ,mode: '0644' } - { src: '{{cfssl_working_dir}}/nomad_ingress-key.pem', dest: '{{nomad_cert_ingress_key}}',mode: '0644' } @@ -183,8 +162,9 @@ - data_center: "nomadder1" - nomad_servers: - "127.0.0.1" + - "{{host_ip}}" - "server.global.nomad" - "localhost" - - "{{nomad_dns}}" - - "server.{{data_center}}.nomad" - - "server.{{host_name}}.{{data_center}}.nomad" + - "{{host_name}}.node.{{data_center}}" + - "{{host_name}}.node.{{data_center}}.consul" + diff --git a/ansible/setup/nomad/tasks/copy_nomad_config_files.yml b/ansible/setup/nomad/tasks/copy_nomad_config_files.yml index b210ba9..2fe9548 100644 --- a/ansible/setup/nomad/tasks/copy_nomad_config_files.yml +++ b/ansible/setup/nomad/tasks/copy_nomad_config_files.yml @@ -23,11 +23,3 @@ dest: "{{nomad_conf_dir}}/client.hcl" owner: "{{ nomad_user }}" register: cfgClient - - - - name: "restart nomad service" - become: true - when: nomad_service.stat.exists and ( (cfgClient is defined and cfgClient.changed) or (cfgServer is defined and cfgServer.changed)) - service: - name: nomad - state: restarted \ No newline at end of file diff --git a/ansible/setup/nomad/tasks/main.yml b/ansible/setup/nomad/tasks/main.yml index 61859b7..2294b28 100644 --- a/ansible/setup/nomad/tasks/main.yml +++ b/ansible/setup/nomad/tasks/main.yml @@ -37,6 +37,52 @@ - name: Nomad service include_tasks: nomad_systemd_install.yml + - name: Reload systemd + become: true + ansible.builtin.systemd: + daemon_reload: yes + + - name: stop nomad service + become: true + failed_when: false + service: + name: nomad + state: stopped + + - name: start nomad service + become: true + service: + name: nomad + state: started + enabled: yes + register: nomadservice + until: nomadservice.status.ActiveState == "active" + retries: 10 + delay: 5 + + - name: Enable Memory Oversubscription + become: false + when: is_master_host + run_once: true + shell: | + curl -s -N \ + --cacert "${NOMAD_CACERT}" \ + --cert "${NOMAD_CLIENT_CERT}" \ + --key "${NOMAD_CLIENT_KEY}" \ + "${NOMAD_ADDR}/v1/operator/scheduler/configuration" |\ + jq '.SchedulerConfig | .SchedulerAlgorithm= "spread" | .MemoryOversubscriptionEnabled=true | .PreemptionConfig.SysBatchSchedulerEnabled=true | .PreemptionConfig.BatchSchedulerEnabled=true | .PreemptionConfig.ServiceSchedulerEnabled=true' |\ + curl \ + --cacert "${NOMAD_CACERT}" \ + --cert "${NOMAD_CLIENT_CERT}" \ + --key "${NOMAD_CLIENT_KEY}" \ + -X PUT $NOMAD_ADDR/v1/operator/scheduler/configuration -d @- + changed_when: false + environment: + - NOMAD_ADDR: "https://localhost:4646" + - NOMAD_CACERT: "{{cluster_intermediate_ca_bundle}}" + - NOMAD_CLIENT_CERT: "{{nomad_cert}}" + - NOMAD_CLIENT_KEY: "{{nomad_cert_key}}" + - name: nomad update when: update_nomad|bool == true block: @@ -49,6 +95,25 @@ - name: Nomad service include_tasks: nomad_systemd_install.yml + - name: Reload systemd + ansible.builtin.systemd: + daemon_reload: yes + + - name: stop nomad service + failed_when: false + service: + name: nomad + state: stopped + + - name: start nomad service + service: + name: nomad + state: started + enabled: yes + register: nomadservice + until: nomadservice.status.ActiveState == "active" + retries: 10 + delay: 5 vars: consul_cert_path: "{{base_cert_dir}}/consul" consul_cert: "{{consul_cert_path}}/consul.pem" @@ -69,5 +134,4 @@ - "server.global.nomad" - "localhost" - "{{nomad_dns}}" - - "server.{{data_center}}.nomad" - - "server.{{host_name}}.{{data_center}}.nomad" \ No newline at end of file + - "{{host_name}}.node.{{data_center}}.consul" \ No newline at end of file diff --git a/ansible/setup/nomad/tasks/nomad_systemd_install.yml b/ansible/setup/nomad/tasks/nomad_systemd_install.yml index 1ef0f98..e1b7eb1 100644 --- a/ansible/setup/nomad/tasks/nomad_systemd_install.yml +++ b/ansible/setup/nomad/tasks/nomad_systemd_install.yml @@ -12,27 +12,32 @@ dest: /etc/systemd/system/nomad.eligtion.service register: serviceconfig_eligtion + - name: Template nomad_kill_pause_containers.sh.j2 + template: + force: yes + src: service/nomad_kill_pause_containers.sh.j2 + dest: "{{nomad_conf_dir}}/nomad_kill_pause_containers.sh" + register: template_kill_pause + - name: Template nomad_node_drain.sh.j2 template: force: yes src: service/nomad_node_drain.sh.j2 dest: "{{nomad_conf_dir}}/nomad_node_drain.sh" + register: template_node_drain - name: "Changing perm of {{nomad_conf_dir}}/nomad_node_drain.sh" file: dest="{{nomad_conf_dir}}/nomad_node_drain.sh" mode=+x + - name: "Changing perm of {{nomad_conf_dir}}/nomad_kill_pause_containers.sh" + file: dest="{{nomad_conf_dir}}/nomad_kill_pause_containers.sh" mode=+x + - name: enable nomad eligtion service service: name: nomad.eligtion enabled: yes - - name: Reload systemd - when: serviceconfig_eligtion.changed - ansible.builtin.systemd: - daemon_reload: yes - - - - name: Template nomad.service.server + - name: Template nomad.service template: force: yes src: service/nomad.service.j2 @@ -40,50 +45,6 @@ register: serviceconfig - - name: stop nomad service - when: serviceconfig.changed - failed_when: false - service: - name: nomad - state: stopped - - - name: Reload systemd - when: serviceconfig.changed - ansible.builtin.systemd: - daemon_reload: yes - - - name: start nomad service - service: - name: nomad - state: started - enabled: yes - register: nomadservice - until: nomadservice.status.ActiveState == "active" - retries: 10 - delay: 5 - - - name: Enable Memory Oversubscription - become: false - when: is_master_host - run_once: true - shell: | - curl -s -N \ - --cacert "${NOMAD_CACERT}" \ - --cert "${NOMAD_CLIENT_CERT}" \ - --key "${NOMAD_CLIENT_KEY}" \ - "${NOMAD_ADDR}/v1/operator/scheduler/configuration" |\ - jq '.SchedulerConfig | .SchedulerAlgorithm= "spread" | .MemoryOversubscriptionEnabled=true | .PreemptionConfig.SysBatchSchedulerEnabled=true | .PreemptionConfig.BatchSchedulerEnabled=true | .PreemptionConfig.ServiceSchedulerEnabled=true' |\ - curl \ - --cacert "${NOMAD_CACERT}" \ - --cert "${NOMAD_CLIENT_CERT}" \ - --key "${NOMAD_CLIENT_KEY}" \ - -X PUT $NOMAD_ADDR/v1/operator/scheduler/configuration -d @- - changed_when: false - environment: - - NOMAD_ADDR: "https://localhost:4646" - - NOMAD_CACERT: "{{cluster_intermediate_ca_bundle}}" - - NOMAD_CLIENT_CERT: "{{nomad_cert}}" - - NOMAD_CLIENT_KEY: "{{nomad_cert_key}}" # failed_when: install.rc > 2 diff --git a/ansible/setup/nomad/templates/cert/nomad_client.j2 b/ansible/setup/nomad/templates/cert/nomad_agent.j2 similarity index 70% rename from ansible/setup/nomad/templates/cert/nomad_client.j2 rename to ansible/setup/nomad/templates/cert/nomad_agent.j2 index c10c3f7..3ab03cf 100644 --- a/ansible/setup/nomad/templates/cert/nomad_client.j2 +++ b/ansible/setup/nomad/templates/cert/nomad_agent.j2 @@ -1,5 +1,5 @@ { - "CN": "client.{{host_name}}.{{data_center}}.nomad", + "CN": "{{host_name}}.node.{{data_center}}.consul", "key": { "algo": "rsa", "size": 4096 @@ -13,5 +13,5 @@ "ST": "{{ca_cert_name_st}}" } ], - "hosts": {{(nomad_servers + masters) | to_json }} + "hosts": {{( nomad_servers ) | to_json }} } \ No newline at end of file diff --git a/ansible/setup/nomad/templates/cert/nomad_server.j2 b/ansible/setup/nomad/templates/cert/nomad_server.j2 deleted file mode 100644 index 1e4cbe2..0000000 --- a/ansible/setup/nomad/templates/cert/nomad_server.j2 +++ /dev/null @@ -1,17 +0,0 @@ -{ - "CN": "server.{{host_name}}.{{data_center}}.nomad", - "key": { - "algo": "rsa", - "size": 4096 - }, - "names": [ - { - "C": "{{ca_cert_name_c}}", - "L": "{{ca_cert_name_l}}", - "O": "{{ca_cert_name_o}}", - "OU": "{{ca_cert_name_ou}}", - "ST": "{{ca_cert_name_st}}" - } - ], - "hosts": {{( nomad_servers + masters) | to_json }} -} \ No newline at end of file diff --git a/ansible/setup/nomad/templates/conf/nomad_client.j2 b/ansible/setup/nomad/templates/conf/nomad_client.j2 index 9819ee4..28f56fa 100644 --- a/ansible/setup/nomad/templates/conf/nomad_client.j2 +++ b/ansible/setup/nomad/templates/conf/nomad_client.j2 @@ -18,10 +18,11 @@ advertise { } client { enabled = true + network_interface = "eth1" meta { node_type= "worker" connect.log_level = "debug" - # connect.sidecar_image: "{{registry_dns}}/envoyproxy/envoy:v{{envoy_version}}" + connect.sidecar_image= "{{registry_dns}}/envoyproxy/envoy:v{{envoy_version}}" } server_join { retry_join = {{masters | to_json }} @@ -30,11 +31,11 @@ client { } # Either leave_on_interrupt or leave_on_terminate must be set # for this to take effect. -# drain_on_shutdown { -# deadline = "2m" -# force = false -# ignore_system_jobs = false -# } + drain_on_shutdown { + deadline = "2m" + force = false + ignore_system_jobs = false + } host_volume "ca_cert" { path = "/usr/local/share/ca-certificates/cloudlocal" read_only = true @@ -50,6 +51,23 @@ client { path = "{{base_cert_dir}}/consul" read_only = true } + + ## Cert consul client + ## Needed for jenkins + ## Should be deleted after resolve https://github.com/suikast42/nomadder/issues/100 + host_volume "cert_nomad" { + path = "{{base_cert_dir}}/nomad" + read_only = true + } + + ## Cert docker client + ## Needed for jenkins + ## Should be deleted after migrating to vault + host_volume "cert_docker" { + path = "{{base_cert_dir}}/docker" + read_only = true + } + host_network "public" { interface = "eth0" #cidr = "203.0.113.0/24" diff --git a/ansible/setup/nomad/templates/service/nomad.service.j2 b/ansible/setup/nomad/templates/service/nomad.service.j2 index 81b3f13..73c5e06 100644 --- a/ansible/setup/nomad/templates/service/nomad.service.j2 +++ b/ansible/setup/nomad/templates/service/nomad.service.j2 @@ -5,8 +5,8 @@ Description=Nomad Documentation=https://www.nomadproject.io/docs/ {% if is_master_host %} -Wants=network-online.target,consul.service -After=network-online.target,consul.service +Wants=network-online.target,containerd.service,docker.service, consul.service +After=network-online.target,containerd.service,docker.service,consul.service {% else %} Wants=network-online.target,containerd.service,docker.service,consul.service After=network-online.target,containerd.service,docker.service,consul.service @@ -36,12 +36,16 @@ Group=root ExecReload=/bin/kill -HUP $MAINPID ExecStart=/usr/local/bin/nomad agent -config {{nomad_conf_dir}} {% if is_worker_host %} +# See issue https://github.com/hashicorp/nomad/issues/17299 +# See issue https://github.com/suikast42/nomadder/issues/138 +ExecStartPre={{nomad_conf_dir}}/nomad_kill_pause_containers.sh # nomad client have a active setting drain_on_shutdown # this drains the node and mark it as ineligible. # Make the node eligible again -#ExecStartPost=/bin/bash -c '(while ! nomad node eligibility -enable -self -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad-cli.pem -client-key=/etc/opt/certs/nomad/nomad-cli-key.pem >/dev/null; do echo "Waiting for port nomad agent runs..."; sleep 1; done); sleep 1' ExecStartPost=systemctl restart nomad.eligtion.service -ExecStop={{nomad_conf_dir}}/nomad_node_drain.sh +# Use node drain over client config drain_on_shutdown +# Enable this section if you disable the option drain_on_shutdown +#ExecStop={{nomad_conf_dir}}/nomad_node_drain.sh {% endif %} KillMode=process diff --git a/ansible/setup/nomad/templates/service/nomad_kill_pause_containers.sh.j2 b/ansible/setup/nomad/templates/service/nomad_kill_pause_containers.sh.j2 new file mode 100644 index 0000000..e95d066 --- /dev/null +++ b/ansible/setup/nomad/templates/service/nomad_kill_pause_containers.sh.j2 @@ -0,0 +1,5 @@ +#!/bin/bash +CONTAINER_IDS=$(docker ps | grep "amd64") +if [ -n "$CONTAINER_IDS" ]; then + docker kill $(docker ps | grep "amd64" | awk '{ print $1 }') +fi \ No newline at end of file diff --git a/ansible/setup/nomad/templates/service/nomad_node_drain.sh.j2 b/ansible/setup/nomad/templates/service/nomad_node_drain.sh.j2 index 5bbb638..05c1719 100644 --- a/ansible/setup/nomad/templates/service/nomad_node_drain.sh.j2 +++ b/ansible/setup/nomad/templates/service/nomad_node_drain.sh.j2 @@ -1,3 +1,3 @@ #!/bin/bash -if [ ! -f "/home/{{ansible_user}}/notdrain" ] ; then nomad node drain -enable -self -deadline "2m" -m "Node shutdown" -yes -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad.pem -client-key=/etc/opt/certs/nomad/nomad-key.pem;fi \ No newline at end of file +if [ ! -f "/home/{{ansible_user}}/notdrain" ] ; then nomad node drain -enable -self -deadline "1h" -m "Node shutdown" -yes -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad.pem -client-key=/etc/opt/certs/nomad/nomad-key.pem;fi \ No newline at end of file diff --git a/ansible/setup/stack_observability/tasks/main.yml b/ansible/setup/stack_observability/tasks/main.yml index 88cbfc1..45ea936 100644 --- a/ansible/setup/stack_observability/tasks/main.yml +++ b/ansible/setup/stack_observability/tasks/main.yml @@ -24,13 +24,14 @@ - docker_build_dir: "/tmp/dockerbuild" - nomad_cli_cert: "{{base_cert_dir}}/nomad/nomad-cli.pem" - nomad_cli_cert_key: "{{base_cert_dir}}/nomad/nomad-cli-key.pem" - - version_grafana: "9.5.2" + - version_grafana: "9.5.3" - version_grafana_nomadder: "{{version_grafana}}.2" - version_grafana_mimir: "2.8.0" - version_grafana_loki: "2.8.2" - version_grafana_tempo: "2.1.1" - - version_nats_server: "2.9.16" - - version_logunifer: "0.1.1" + - version_nats_server: "2.9.18" + - version_logunifer: "0.1.2" + - version_blackbox_exporter: "0.24.0" - version_nats_prometheus_exporter: "0.11.0" - version_nats_prometheus_exporter_nomadder: "{{version_nats_prometheus_exporter}}.0" - stack_deployment_dir: "{{master_0_job_dir}}/observability" \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Cluster overview-1676113506033.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Cluster overview-1676113506033.json deleted file mode 100644 index 76015cb..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Cluster overview-1676113506033.json +++ /dev/null @@ -1,2140 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Nomad cluster dashboard. Using built in nomad's exporter and loki. Example cluster is present there https://github.com/theztd/startup-infra-docker in ", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 17549, - "graphTooltip": 1, - "id": 15, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 207, - "panels": [], - "title": "Cluster Summary", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 0, - "y": 1 - }, - "id": 221, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_status_running{}", - "instant": true, - "interval": "", - "legendFormat": "Running", - "refId": "A" - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "In the whole cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 2, - "y": 1 - }, - "id": 107, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.0.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(nomad_client_allocs_cpu_allocated) by (exported_job)", - "interval": "", - "legendFormat": "{{exported_job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(nomad_client_unallocated_cpu)", - "hide": false, - "legendFormat": "Free cluster CPU", - "range": true, - "refId": "B" - } - ], - "title": "Allocated CPU ($datacenter)", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "links": [ - { - "title": "Set time range", - "url": "https://mon.cinarra.com/d/4NYOvS-7z/nomad?orgId=1&var-env=$env&var-alloc_id=${__data.fields.alloc_id}&from=${__data.fields.from}&to=${__data.fields.to}" - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "from" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "to" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alloc_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 287 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Kills" - }, - "properties": [ - { - "id": "custom.width", - "value": 45 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 131 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "task" - }, - "properties": [ - { - "id": "custom.width", - "value": 92 - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 13, - "x": 8, - "y": 1 - }, - "id": 215, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": true, - "expr": "nomad_client_allocs_oom_killed{}", - "format": "table", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "oom", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": true, - "expr": "timestamp(nomad_client_allocs_oom_killed{}) * 1000 - 3600000", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "from", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": true, - "expr": "timestamp(nomad_client_allocs_oom_killed{}) * 1000 + 3600000", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "to", - "refId": "C" - } - ], - "title": "OOM kills", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "alloc_id" - } - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "alloc_id|exporter_job 1|Time 1|instance 1|namespace 1|task 1|task_group 1|Value .*" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Value #B": true, - "Value #C": true, - "namespace 1": true, - "task_group 1": true - }, - "indexByName": { - "Time 1": 0, - "Value #A": 6, - "Value #B": 7, - "Value #C": 8, - "alloc_id": 3, - "instance 1": 1, - "namespace 1": 4, - "task 1": 2, - "task_group 1": 5 - }, - "renameByName": { - "Time 1": "Time", - "Value #A": "Kills", - "Value #B": "from", - "Value #C": "to", - "alloc_id": "", - "instance 1": "Instance", - "namespace 1": "namespace", - "task 1": "task", - "task_group 1": "group" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Time" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Unhealthy" - }, - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 229, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "up{job=\"integrations/nomad\"}", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Node status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-orange", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 0, - "y": 4 - }, - "id": 217, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_status_pending", - "instant": true, - "interval": "", - "legendFormat": "Pending", - "refId": "A" - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Run" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - }, - { - "id": "custom.width", - "value": 64 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Complete" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" - } - }, - { - "id": "custom.width", - "value": 65 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "color", - "value": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - } - }, - { - "id": "custom.width", - "value": 63 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Starting" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "color", - "value": { - "fixedColor": "text", - "mode": "fixed" - } - }, - { - "id": "custom.width", - "value": 89 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Quequed" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "color", - "value": { - "fixedColor": "transparent", - "mode": "fixed" - } - }, - { - "id": "custom.width", - "value": 94 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Lost" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "color", - "value": { - "fixedColor": "#000000", - "mode": "fixed" - } - }, - { - "id": "custom.width", - "value": 62 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.width", - "value": 264 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Node" - }, - "properties": [ - { - "id": "custom.width", - "value": 187 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Job" - }, - "properties": [ - { - "id": "custom.width", - "value": 351 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 13, - "x": 8, - "y": 5 - }, - "id": 231, - "options": { - "footer": { - "fields": [ - "Value #F", - "Value #A", - "Value #B", - "Value #D", - "Value #E", - "Value #C" - ], - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_summary_running{}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "running", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_summary_failed{}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "failed", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "exemplar": false, - "expr": "nomad_nomad_job_summary_lost{}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "lost", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_summary_starting{}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "starting", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_summary_queued{}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "queued", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_summary_complete{}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "complete", - "range": false, - "refId": "F" - } - ], - "title": "Allocations", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "task_group" - } - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "task_group", - "Value #A", - "Value #B", - "Value #C", - "host 1" - ], - "pattern": "exported_job 1|namespace 1|task_group|Value .*" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "exported_job 1": false, - "host": true, - "host 1": true, - "namespace 1": true, - "task_group": false - }, - "indexByName": { - "Value #A": 4, - "Value #B": 5, - "Value #C": 6, - "exported_job 1": 0, - "host 1": 1, - "namespace 1": 3, - "task_group": 2 - }, - "renameByName": { - "Value #A": "Run", - "Value #B": "Failed", - "Value #C": "Lost", - "Value #D": "Starting", - "Value #E": "Quequed", - "Value #F": "Complete", - "exported_job 1": "Job", - "host 1": "Node", - "namespace 1": "namespace", - "task_group": "group", - "task_group 1": "Group" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "super-light-red", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 0, - "y": 7 - }, - "id": 219, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_nomad_job_status_dead", - "instant": true, - "interval": "", - "legendFormat": "Dead", - "refId": "A" - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "In whole cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 2, - "y": 7 - }, - "id": 86, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.0.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(nomad_client_allocs_memory_allocated) by (exported_job)", - "interval": "", - "legendFormat": "{{exported_job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(nomad_client_host_memory_free)", - "hide": false, - "legendFormat": "Free cluster MEM", - "range": true, - "refId": "B" - } - ], - "title": "Allocated MEM ($datacenter)", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 0, - "y": 10 - }, - "id": 248, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": false, - "expr": "nomad_client_allocs_oom_killed", - "instant": true, - "interval": "", - "legendFormat": "Killed", - "range": false, - "refId": "A" - } - ], - "transformations": [], - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 2, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Node Status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "displayName": "($host)", - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-red", - "value": null - }, - { - "color": "light-orange", - "value": 360 - }, - { - "color": "light-green", - "value": 86400 - } - ] - }, - "unit": "dtdurations" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 14 - }, - "id": 4, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "repeat": "host", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "nomad_client_uptime{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "$host", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 3, - "y": 14 - }, - "id": 11, - "links": [], - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "repeat": "host", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "expr": "nomad_client_host_memory_free{datacenter=~\"$datacenter\", host=~\"$host\"} /nomad_client_host_memory_total{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Total", - "range": true, - "refId": "B" - } - ], - "title": "Free memmory", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "light-red", - "value": null - }, - { - "color": "light-orange", - "value": 15 - }, - { - "color": "light-green", - "value": 30 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 9, - "y": 14 - }, - "id": 13, - "links": [], - "options": { - "displayMode": "lcd", - "minVizHeight": 0, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.3.6", - "repeat": "host", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "expr": "avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\" ,disk=~\"/dev/sd.*\"}) by (disk) / max(nomad_client_host_disk_size{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk) ", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{disk}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "expr": "avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk)", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "Available - {{disk}}", - "range": true, - "refId": "A" - } - ], - "title": "Free Disk Space ($host)", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 14, - "y": 14 - }, - "id": 58, - "links": [], - "options": { - "displayMode": "lcd", - "minVizHeight": 2, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.3.6", - "repeat": "host", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "nomad_client_allocations_migrating{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Migrating", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "nomad_client_allocations_blocked{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Blocked", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "nomad_client_allocations_pending{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pending", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "expr": "nomad_client_allocations_running{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Running", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "nomad_client_allocations_terminal{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Terminal", - "refId": "E" - } - ], - "title": "Allocation status", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "$host", - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "light-red", - "value": null - }, - { - "color": "#EAB839", - "value": 10 - }, - { - "color": "light-green", - "value": 50 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 18, - "y": 14 - }, - "id": 33, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "repeat": "host", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "expr": "nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"})", - "format": "time_series", - "intervalFactor": 1, - "range": true, - "refId": "A" - } - ], - "title": "Unalocated CPU", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "$host", - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "light-red", - "value": null - }, - { - "color": "#EAB839", - "value": 10 - }, - { - "color": "light-green", - "value": 50 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 14 - }, - "id": 40, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "repeat": "host", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Unalocated Mem", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 247, - "panels": [], - "title": "RAFT status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "Time difference between when a message was first passed into Raft and when the resulting commit took place on the leader", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 223, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "exemplar": true, - "expr": "nomad_raft_commitTime{quantile=\"0.9\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "title": "Raft Commit Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "Number of Raft transactions - general indicator of the write load on the servers.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 225, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "editorMode": "code", - "exemplar": true, - "expr": "increase(nomad_raft_apply{}[$__rate_interval])", - "interval": "", - "legendFormat": "{{ instance }}", - "range": true, - "refId": "A" - } - ], - "title": "Raft Apply Count", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 239, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 127, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "{job=\"nomad.service\"} | json | line_format `{{.service_name}}@{{.service_node_name}} [{{.log_levelEmoji}} {{.level}}] [{{.tags}}] {{.message}}`", - "hide": false, - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs ($datacenter)", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "nomad", - "host", - "cluster" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "nomadder1", - "value": "nomadder1" - }, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "definition": "", - "hide": 0, - "includeAll": false, - "label": "DC", - "multi": false, - "name": "datacenter", - "options": [], - "query": { - "query": "label_values(nomad_client_uptime, datacenter)", - "refId": "de1-datacenter-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Host", - "multi": true, - "name": "host", - "options": [], - "query": { - "query": "label_values(nomad_client_uptime{datacenter=~\"$datacenter\"}, host)", - "refId": "de1-host-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Cluster overview", - "uid": "masiNomadCluster", - "version": 8, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/LogsTest-1680298618653.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/LogsTest-1680298618653.json deleted file mode 100644 index c9009b9..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/LogsTest-1680298618653.json +++ /dev/null @@ -1,1010 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 7, - "panels": [], - "title": "Log level distribution for job type $__interval", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "error " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "debug " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "fatal " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "info " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trace " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "unknown " - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "warn " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.7", - "repeat": "job_types", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "sum by(level) (count_over_time({ingress=\"$ingress\",job_type=~\"$job_types\"} [$__interval]))", - "key": "Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0", - "legendFormat": "{{level}} ", - "queryType": "range", - "refId": "A" - } - ], - "title": "$job_types ", - "transformations": [], - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "error " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "debug " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "fatal " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "info " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trace " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "unknown " - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "warn " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 5, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": false, - "values": [] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "sum by(level) (count_over_time({ingress=\"$ingress\"} [1w]))", - "key": "Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0", - "legendFormat": "{{level}} ", - "queryType": "range", - "refId": "A" - } - ], - "title": "Log distrubution", - "transformations": [], - "transparent": true, - "type": "piechart" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "debug " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "error " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "fatal " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "info " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trace" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "unknown " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "text", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "warn " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 19, - "x": 5, - "y": 10 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "sum by(level) (count_over_time({ingress=\"$ingress\"} [$__interval]))", - "key": "Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0", - "legendFormat": "{{level}} ", - "queryType": "range", - "refId": "A" - } - ], - "title": "Log distrubution in $__interval", - "transformations": [ - { - "disabled": true, - "id": "labelsToFields", - "options": { - "mode": "columns", - "valueLabel": "level" - } - } - ], - "transparent": true, - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 9, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-orange", - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "color-text" - }, - "filterable": true, - "inspect": false - }, - "mappings": [ - { - "options": { - "pattern": ".*", - "result": { - "color": "super-light-blue", - "index": 0 - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "level" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "thresholds" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "debug": { - "color": "dark-blue", - "index": 3 - }, - "error": { - "color": "semi-dark-red", - "index": 0 - }, - "fatal": { - "color": "dark-red", - "index": 1 - }, - "info": { - "color": "dark-green", - "index": 2 - }, - "trace": { - "color": "dark-purple", - "index": 4 - }, - "unknown": { - "color": "text", - "index": 5 - }, - "warn": { - "color": "dark-orange", - "index": 6 - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 5, - "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Count (sum)" - } - ] - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "sum by(job_type, stack,job,level) (count_over_time({ingress=\"$ingress\"} [1w] ))", - "key": "Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0", - "legendFormat": "", - "queryType": "instant", - "refId": "A", - "resolution": 1 - } - ], - "title": " ", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Time": 0, - "Value #A": 5, - "job": 3, - "job_type": 1, - "level": 4, - "stack": 2 - }, - "renameByName": { - "Time": "", - "Value #A": "Count", - "job_type": "Job Type", - "level": "" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "Count": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Job Type": { - "aggregations": [], - "operation": "groupby" - }, - "job": { - "aggregations": [], - "operation": "groupby" - }, - "job_type": { - "aggregations": [], - "operation": "groupby" - }, - "level": { - "aggregations": [], - "operation": "groupby" - }, - "stack": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Job Type" - } - ] - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "job" - } - ] - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Count (sum)" - } - ] - } - } - ], - "transparent": true, - "type": "table" - } - ], - "title": "Row title", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 15, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 25, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 13, - "options": { - "dedupStrategy": "exact", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "builder", - "expr": "{ingress=\"$ingress\", job_type=~\"$job_types\"} |= `` | json | line_format `{{.job}}@{{.host_name}} [{{.log_levelEmoji}} {{.level}}] [{{.tags}}] {{.message}}`", - "key": "Q-eef4e925-e736-47be-9448-c35f4c6acf7d-0", - "queryType": "range", - "refId": "A" - } - ], - "title": "New Panel", - "transparent": true, - "type": "logs" - } - ], - "title": "Row title", - "type": "row" - } - ], - "refresh": "5s", - "revision": 1, - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "filters": [], - "hide": 0, - "name": "Filters", - "skipUrlSync": false, - "type": "adhoc" - }, - { - "current": { - "selected": false, - "text": "ingress.logs.journald", - "value": "ingress.logs.journald" - }, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "definition": "label_values(job_type)", - "hide": 0, - "includeAll": false, - "label": "ingress", - "multi": false, - "name": "ingress", - "options": [], - "query": { - "label": "ingress", - "refId": "LokiVariableQueryEditor-VariableQuery", - "stream": "", - "type": 1 - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "definition": "label_values(job_type)", - "hide": 0, - "includeAll": true, - "label": "job_types", - "multi": true, - "name": "job_types", - "options": [], - "query": { - "label": "job_type", - "refId": "LokiVariableQueryEditor-VariableQuery", - "stream": "{ingress=\"$ingress\"}", - "type": 1 - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "1s", - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "LogsTest", - "uid": "KMg_v90Vz", - "version": 3, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Loki2.0 Global Metrics-1675942743100.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Loki2.0 Global Metrics-1675942743100.json deleted file mode 100644 index d4f3115..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/Loki2.0 Global Metrics-1675942743100.json +++ /dev/null @@ -1,3595 +0,0 @@ -{ - "annotations": { - "list": [ - { - "$$hashKey": "object:7", - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Loki metrics via 2.0", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 13407, - "graphTooltip": 0, - "id": 14, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "", - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 21, - "interval": "", - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "
\n \n Loki Global Metrics\n
", - "mode": "html" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "loki_build_info", - "format": "table", - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 4, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 2 - }, - "id": 59, - "interval": "", - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "/^version$/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "loki_build_info", - "format": "table", - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Loki Version", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 2 - }, - "id": 10, - "interval": "", - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(log_messages_total)", - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Message Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 2 - }, - "id": 44, - "interval": "", - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "cortex_prometheus_notifications_alertmanagers_discovered", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Current Alerts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 9, - "y": 2 - }, - "id": 48, - "interval": "", - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "cortex_prometheus_notifications_sent_total", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Alerts Event Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 4, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 12, - "y": 2 - }, - "id": 23, - "interval": "", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(loki_store_series_total)", - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Store Series Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 4, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 15, - "y": 2 - }, - "id": 41, - "interval": "", - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(loki_ingester_chunk_stored_bytes_total)", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Store Chunks Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 18, - "y": 2 - }, - "id": 49, - "interval": "", - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(cortex_prometheus_rule_group_rules)", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Rules Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 4, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 2 - }, - "id": 24, - "interval": "", - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "loki_panic_total", - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Panic ", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 5, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 8, - "interval": "", - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(irate(log_messages_total[1m])) by (level)", - "interval": "", - "legendFormat": "{{ level }}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Messages Input", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 53, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(cortex_s3_request_duration_seconds_bucket[5m])) by (le,operation))", - "interval": "", - "legendFormat": "{{operation}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 Request Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "hiddenSeries": false, - "id": 11, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket[5m])) by (le,route))", - "interval": "", - "legendFormat": "{{ route }}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "API Request Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "hiddenSeries": false, - "id": 12, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket[5m])) by (le,type))", - "interval": "", - "legendFormat": "{{ type }}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "LogQL Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 13, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket[5m])) by (le,type))", - "interval": "", - "legendFormat": "{{ type }}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "LogQL Processed bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 43, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Ruler", - "type": "row" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 24 - }, - "id": 46, - "interval": "", - "legend": { - "show": true, - "values": true - }, - "legendType": "Right side", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "pieType": "pie", - "pluginVersion": "7.3.1", - "strokeWidth": 1, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "cortex_prometheus_rule_evaluations_total", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Evaluations", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "cortex_prometheus_notifications_sent_total", - "instant": true, - "interval": "1m", - "legendFormat": "Notifications", - "refId": "B" - } - ], - "title": "Rules Status ", - "type": "grafana-piechart-panel", - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 7, - "y": 24 - }, - "hiddenSeries": false, - "id": 28, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(rate(cortex_prometheus_notifications_latency_seconds[5m])) by (quantile)", - "interval": "", - "legendFormat": "TP {{quantile}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Notifications Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 24 - }, - "hiddenSeries": false, - "id": 47, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(rate(cortex_prometheus_rule_evaluation_duration_seconds[5m])) by (quantile)", - "interval": "", - "legendFormat": "TP {{quantile}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Rule Evaluations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 34, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Cache ", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 45, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le,name,method))", - "interval": "", - "legendFormat": "{{method}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cache Request Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 29, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le,name,method))", - "interval": "", - "legendFormat": "{{name}} / {{method}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cache Value Size bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 10, - "x": 0, - "y": 37 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "rate(cortex_cache_hits[5m])", - "interval": "", - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hits Keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 10, - "y": 37 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "cortex_cache_background_queue_length", - "interval": "", - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Background Queue Length", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 17, - "y": 37 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "cortex_cache_fetched_keys", - "interval": "", - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Fetched Keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 36, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Index", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 44 - }, - "hiddenSeries": false, - "id": 38, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(cortex_cassandra_request_duration_seconds_bucket[5m])) by (le,operation))", - "interval": "", - "legendFormat": "{{operation}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cassandra Request Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 44 - }, - "hiddenSeries": false, - "id": 39, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(cortex_chunk_store_chunks_per_query_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "chunks", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunk Store Chunks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 44 - }, - "hiddenSeries": false, - "id": 54, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(cortex_table_manager_sync_duration_seconds_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Table-Manager Sync Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "decimals": 2, - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 19, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Distributor", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 51 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum by (tenant) (rate(loki_distributor_lines_received_total[5m]))", - "interval": "", - "legendFormat": "distributor / {{ tenant}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Received Lines / sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 51 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(rate(loki_distributor_bytes_received_total[5m])) by (tenant)", - "interval": "", - "legendFormat": "distributor / {{tenant}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Distributor Received bytes / sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:837", - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:838", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 51 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "rate(loki_distributor_ingester_appends_total[5m])", - "interval": "", - "legendFormat": "successed", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "loki_distributor_ingester_append_failures_total", - "interval": "", - "legendFormat": "failed", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "batch appends sent to ingesters / sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [], - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 51 - }, - "hiddenSeries": false, - "id": 22, - "interval": "", - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum by (status) (rate(loki_store_series_total[5m]))", - "instant": false, - "interval": "", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store Series / sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:88", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:89", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 57 - }, - "id": 66, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Ingster", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 58 - }, - "hiddenSeries": false, - "id": 71, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "blocks", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blocks / Chunk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 58 - }, - "hiddenSeries": false, - "id": 68, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "chunk size", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunk Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 58 - }, - "hiddenSeries": false, - "id": 72, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "ages", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunk Age", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 64 - }, - "hiddenSeries": false, - "id": 70, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "ratio", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunk Compression Ratios", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "percent", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 64 - }, - "hiddenSeries": false, - "id": 69, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "durations.", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunk Encode Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 64 - }, - "hiddenSeries": false, - "id": 73, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket[5m])) by (le))", - "interval": "", - "legendFormat": "lines", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Lines / Chunk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:155", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:156", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 70 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "rate(loki_ingester_chunks_created_total[5m])", - "interval": "", - "legendFormat": "create", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total[5m]))", - "interval": "", - "legendFormat": "flush / {{ reason }}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum by (fake) (rate(loki_ingester_chunks_stored_total[5m]))", - "interval": "", - "legendFormat": "stored", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "", - "interval": "", - "legendFormat": "", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunk Status", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 70 - }, - "hiddenSeries": false, - "id": 61, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "loki_ingester_memory_chunks", - "interval": "", - "legendFormat": "chunks", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunks in Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 70 - }, - "hiddenSeries": false, - "id": 62, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum by (tenan) (loki_ingester_memory_streams)", - "interval": "", - "legendFormat": "streams", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Chunks in Streams", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 76 - }, - "hiddenSeries": false, - "id": 67, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "rate(loki_ingester_streams_created_total[5m])", - "interval": "", - "legendFormat": "create", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "rate(loki_ingester_streams_removed_total[5m])", - "interval": "", - "legendFormat": "delete", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Strams Stats", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1092", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1093", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "loki" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "1s", - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Loki2.0 Global Metrics", - "uid": "MQHVDmtWk", - "version": 2, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/consul-exporter-dashboard_rev2.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/consul-exporter-dashboard_rev2.json deleted file mode 100644 index a037705..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/consul-exporter-dashboard_rev2.json +++ /dev/null @@ -1,570 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.0.1" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Consul Exporter Dashboar for Prometheus", - "editable": true, - "gnetId": 12049, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 2, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "consul_catalog_services", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{app}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total Services", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "consul_raft_peers", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "peers", - "refId": "A" - }, - { - "expr": "consul_raft_leader", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "leader", - "refId": "B" - }, - { - "expr": "consul_serf_lan_members", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "serf_lan_members", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Consul Peers and Leaders", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "fill": 1, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (service_name, status) (consul_health_service_status{status!=\"passing\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{service_name}} - {{status}}", - "refId": "A" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0, - "yaxis": "left" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Unhealthy Consul Services", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "consul_health_node_status{status!='passing'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{node}} - {{status}}", - "refId": "A" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0, - "yaxis": "left" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Unhealthy Consul Nodes", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "fill": 1, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (node,service_id,status) (consul_health_service_status)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{node}} - {{service_id}} - {{status}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Consul Services and Checks", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": false, - "schemaVersion": 18, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Consul Exporter Dashboard", - "uid": "g7j0xhjZk", - "version": 3 -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetsteram-dash.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetsteram-dash.json deleted file mode 100644 index c4463db..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetsteram-dash.json +++ /dev/null @@ -1,1405 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "NATS JetStream Dashboard.", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 14725, - "graphTooltip": 1, - "id": 6, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 3, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.75 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 28, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_jetstream_stats_storage{server_id=~\"$server\"})/sum(gnatsd_varz_jetstream_config_max_storage{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Storage Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 4, - "y": 0 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_jetstream_stats_storage{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Storage Used", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "decimals": 3, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.75 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 9, - "y": 0 - }, - "id": 31, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_jetstream_stats_memory{server_id=~\"$server\"})/sum(gnatsd_varz_jetstream_config_max_memory{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Memory Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 13, - "y": 0 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_jetstream_stats_memory{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory Used", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_connections{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 4, - "y": 3 - }, - "id": 30, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_jetstream_config_max_storage{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Max Storage", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 13, - "y": 3 - }, - "id": 33, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(gnatsd_varz_jetstream_config_max_memory{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Memory", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 3 - }, - "id": 29, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(jetstream_server_total_consumers{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Consumers", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 19, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Stream metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 7 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(jetstream_stream_total_bytes) by (stream_name)", - "interval": "", - "legendFormat": "{{stream_name}}", - "refId": "A" - } - ], - "title": "Stream data size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 7 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(jetstream_stream_total_messages) by (stream_name)", - "interval": "", - "legendFormat": "{{stream_name}}", - "refId": "A" - } - ], - "title": "Stream message count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 7 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(rate(jetstream_stream_total_messages{server_id=~\"$server\",stream_name=~\"$stream\"}[$__rate_interval])) by (stream_name)", - "hide": false, - "interval": "", - "legendFormat": "{{stream_name}}", - "refId": "A" - } - ], - "title": "Message Rate (per second)", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 23, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "refId": "A" - } - ], - "title": "Consumer Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "description": "Messages added & processed per minute per consumer", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(rate(jetstream_consumer_num_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}[$__rate_interval])+rate(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}[$__rate_interval])) by (consumer_name)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{consumer_name}} +", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "- sum(rate(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\",consumer_name=~\"$consumer\"}[$__rate_interval])) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}} -", - "refId": "A" - } - ], - "title": "Messages per second (++/--)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 21, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}}", - "refId": "A" - } - ], - "title": "Total delivered messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(jetstream_consumer_num_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}}", - "refId": "A" - } - ], - "title": "Pending messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "expr": "sum(jetstream_consumer_num_ack_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}}", - "refId": "A" - } - ], - "title": "Message Acks Pending", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "definition": "label_values(jetstream_server_total_streams, server_id)", - "hide": 0, - "includeAll": true, - "label": "Server", - "multi": true, - "name": "server", - "options": [], - "query": { - "query": "label_values(jetstream_server_total_streams, server_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "definition": "label_values(jetstream_stream_last_seq, stream_name)", - "hide": 0, - "includeAll": true, - "label": "Stream", - "multi": true, - "name": "stream", - "options": [], - "query": { - "query": "label_values(jetstream_stream_last_seq, stream_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "mimir" - }, - "definition": "label_values(jetstream_consumer_num_pending, consumer_name)", - "hide": 0, - "includeAll": true, - "label": "Consumer", - "multi": true, - "name": "consumer", - "options": [], - "query": { - "query": "label_values(jetstream_consumer_num_pending, consumer_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "NATS JetStream", - "uid": "yQUo5l17k", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetstream_rev2.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetstream_rev2.json deleted file mode 100644 index 6d2ed83..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-jetstream_rev2.json +++ /dev/null @@ -1,1238 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS__NATS-PROMETHEUS", - "label": " NATS-Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": [], - "__requires": [ - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.3.4" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "NATS JetStream Dashboard", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "iteration": 1642544826470, - "links": [], - "liveNow": false, - "panels": [ - { - "fieldConfig": { - "defaults": { - "decimals": 3, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.75 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 28, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_jetstream_stats_storage{server_id=~\"$server\"})/sum(gnatsd_varz_jetstream_config_max_storage{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Storage Used", - "type": "gauge" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 4, - "y": 0 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_jetstream_stats_storage{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Storage Used", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "decimals": 3, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.75 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 9, - "y": 0 - }, - "id": 31, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_jetstream_stats_memory{server_id=~\"$server\"})/sum(gnatsd_varz_jetstream_config_max_memory{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Memory Used", - "type": "gauge" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 13, - "y": 0 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_jetstream_stats_memory{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory Used", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_connections{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Connections", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 4, - "y": 3 - }, - "id": 30, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_jetstream_config_max_storage{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Max Storage", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 13, - "y": 3 - }, - "id": 33, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(gnatsd_varz_jetstream_config_max_memory{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Memory", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 3 - }, - "id": 29, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(jetstream_server_total_consumers{server_id=~\"$server\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Consumers", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 19, - "panels": [], - "title": "Stream metrics", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 7 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(jetstream_stream_total_bytes) by (stream_name)", - "interval": "", - "legendFormat": "{{stream_name}}", - "refId": "A" - } - ], - "title": "Stream data size", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 7 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(jetstream_stream_total_messages) by (stream_name)", - "interval": "", - "legendFormat": "{{stream_name}}", - "refId": "A" - } - ], - "title": "Stream message count", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 7 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(jetstream_stream_total_messages{server_id=~\"$server\",stream_name=~\"$stream\"}[$__rate_interval])) by (stream_name)", - "hide": false, - "interval": "", - "legendFormat": "{{stream_name}}", - "refId": "A" - } - ], - "title": "Message Rate (per second)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 23, - "panels": [], - "title": "Consumer Metrics", - "type": "row" - }, - { - "description": "Messages added & processed per minute per consumer", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(jetstream_consumer_num_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}[$__rate_interval])+rate(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}[$__rate_interval])) by (consumer_name)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{consumer_name}} +", - "refId": "B" - }, - { - "expr": "- sum(rate(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\",consumer_name=~\"$consumer\"}[$__rate_interval])) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}} -", - "refId": "A" - } - ], - "title": "Messages per second (++/--)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 21, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}}", - "refId": "A" - } - ], - "title": "Total delivered messages", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(jetstream_consumer_num_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}}", - "refId": "A" - } - ], - "title": "Pending messages", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(jetstream_consumer_num_ack_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)", - "hide": false, - "interval": "", - "legendFormat": "{{consumer_name}}", - "refId": "A" - } - ], - "title": "Message Acks Pending", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 34, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": {}, - "definition": "label_values(jetstream_server_total_streams, server_id)", - "hide": 0, - "includeAll": true, - "label": "Server", - "multi": true, - "name": "server", - "options": [], - "query": { - "query": "label_values(jetstream_server_total_streams, server_id)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false, - "datasource": "${DS__NATS-PROMETHEUS}" - }, - { - "current": {}, - "definition": "label_values(jetstream_stream_last_seq, stream_name)", - "hide": 0, - "includeAll": true, - "label": "Stream", - "multi": true, - "name": "stream", - "options": [], - "query": { - "query": "label_values(jetstream_stream_last_seq, stream_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false, - "datasource": "${DS__NATS-PROMETHEUS}" - }, - { - "current": {}, - "definition": "label_values(jetstream_consumer_num_pending, consumer_name)", - "hide": 0, - "includeAll": true, - "label": "Consumer", - "multi": true, - "name": "consumer", - "options": [], - "query": { - "query": "label_values(jetstream_consumer_num_pending, consumer_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false, - "datasource": "${DS__NATS-PROMETHEUS}" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "NATS JetStream", - "uid": "yQUo5l17k", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-servers_rev1.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-servers_rev1.json deleted file mode 100644 index 9fbaee4..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nats-servers_rev1.json +++ /dev/null @@ -1,965 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS__NATS-PROMETHEUS", - "label": " NATS-Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.3.4" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "NATS Server Dashboard", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 10, - "panels": [], - "title": "OS Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 1, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_cpu", - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_cpu", - "refId": "A", - "step": 4 - } - ], - "title": "Server CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 3, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_mem", - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_mem", - "refId": "A", - "step": 4 - } - ], - "title": "Server Memory", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 11, - "panels": [], - "title": "Throughput", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 9 - }, - "id": 7, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_in_bytes", - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_in_bytes", - "refId": "A", - "step": 10 - } - ], - "title": "Bytes In", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 9 - }, - "id": 8, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_in_msgs", - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_in_msgs", - "refId": "A", - "step": 10 - } - ], - "title": "NATS Msgs In", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 9 - }, - "id": 5, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_out_bytes", - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_out_bytes", - "refId": "A", - "step": 10 - } - ], - "title": "Bytes Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 9 - }, - "id": 6, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_out_msgs", - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_out_msgs", - "refId": "A", - "step": 10 - } - ], - "title": "NATS Msgs Out", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 12, - "panels": [], - "title": "Client Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 2, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_connections", - "intervalFactor": 1, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_connections", - "refId": "A", - "step": 2 - } - ], - "title": "Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 4, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_subscriptions", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_subscriptions", - "refId": "A", - "step": 4 - } - ], - "title": "Subscriptions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS__NATS-PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 9, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "gnatsd_varz_slow_consumers", - "intervalFactor": 1, - "legendFormat": "{{server_id}}", - "metric": "gnatsd_varz_slow_consumers", - "refId": "A", - "step": 2 - } - ], - "title": "Slow Consumers", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 34, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "NATS Servers", - "uid": "Usc6F_1nk", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/node-exporter-full.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/node-exporter-full.json deleted file mode 100644 index 8b2bd86..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/node-exporter-full.json +++ /dev/null @@ -1,23190 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "9.2.3" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "$$hashKey": "object:1058", - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 1860, - "graphTooltip": 0, - "id": null, - "links": [ - { - "icon": "external link", - "tags": [], - "targetBlank": true, - "title": "GitHub", - "type": "link", - "url": "https://github.com/rfmoz/grafana-dashboards" - }, - { - "icon": "external link", - "tags": [], - "targetBlank": true, - "title": "Grafana", - "type": "link", - "url": "https://grafana.com/grafana/dashboards/1860" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 261, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Quick CPU / Mem / Disk", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Busy state of all CPU cores together", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 20, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "range": true, - "refId": "A", - "step": 240 - } - ], - "title": "CPU Busy", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Busy state of all CPU cores together (5 min average)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 155, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "avg(node_load5{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "Sys Load (5m avg)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Busy state of all CPU cores together (15 min average)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 19, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "avg(node_load15{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", - "hide": false, - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "Sys Load (15m avg)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Non available RAM memory", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 1 - }, - "hideTimeOverride": false, - "id": 16, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "refId": "B", - "step": 240 - } - ], - "title": "RAM Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Used Swap", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 25 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 21, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "SWAP Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Used Root FS", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 154, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "Root FS Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of CPU cores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 18, - "y": 1 - }, - "id": 14, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 240 - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "System uptime", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 20, - "y": 1 - }, - "hideTimeOverride": true, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total RootFS", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 18, - "y": 3 - }, - "id": 23, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "RootFS Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total RAM", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 20, - "y": 3 - }, - "id": 75, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "RAM Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total SWAP", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 22, - "y": 3 - }, - "id": 18, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "SWAP Total", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 263, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Basic CPU / Mem / Net / Disk", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Basic CPU info", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy System" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy User" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy Other" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 77, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 250 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Busy System", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Busy User", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy Iowait", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy IRQs", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy Other", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Idle", - "range": true, - "refId": "F", - "step": 240 - } - ], - "title": "CPU Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Basic memory usage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SWAP Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Cache + Buffer" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avaliable" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#DEDAF7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 78, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Total", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Used", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Cache + Buffer", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Free", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SWAP Used", - "refId": "E", - "step": 240 - } - ], - "title": "Memory Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Basic network info per interface", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 74, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "recv {{device}}", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "trans {{device}} ", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Disk space used of all filesystems mounted", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 152, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", - "refId": "A", - "step": 240 - } - ], - "title": "Disk Space Used Basic", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 265, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "percentage", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Idle - Waiting for something to happen" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Iowait - Waiting for I/O to complete" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Irq - Servicing interrupts" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nice - Niced processes executing in user mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Softirq - Servicing softirqs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Steal - Time spent in other operating systems when running in a virtualized environment" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCE2DE", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "System - Processes executing in kernel mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "User - Normal processes executing in user mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#5195CE", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 3, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 250 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "System - Processes executing in kernel mode", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "User - Normal processes executing in user mode", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Nice - Niced processes executing in user mode", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Iowait - Waiting for I/O to complete", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Irq - Servicing interrupts", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Softirq - Servicing softirqs", - "range": true, - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", - "range": true, - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Idle - Waiting for something to happen", - "range": true, - "refId": "J", - "step": 240 - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap - Swap memory usage" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused - Free memory unassigned" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Hardware Corrupted - *./" - }, - "properties": [ - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 24, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Apps - Memory used by user-space applications", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Cache - Parked file data (file content) cache", - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Buffers - Block device (e.g. harddisk) cache", - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Unused - Free memory unassigned", - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Swap - Swap space used", - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", - "refId": "I", - "step": 240 - } - ], - "title": "Memory Stack", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bits out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 84, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 156, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", - "refId": "A", - "step": 240 - } - ], - "title": "Disk Space Used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 229, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{device}} - Reads completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Writes completed", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*read*./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 42, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Successfully read bytes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Successfully written bytes", - "refId": "B", - "step": 240 - } - ], - "title": "I/O Usage Read / Write", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "%util", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 127, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}}", - "refId": "A", - "step": 240 - } - ], - "title": "I/O Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "percentage", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^Guest - /" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#5195ce", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^GuestNice - /" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#c15c17", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 43 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", - "range": true, - "refId": "B" - } - ], - "title": "CPU spent seconds in guests (VMs)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "CPU / Memory / Net / Disk", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 266, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 136, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Active / Inactive", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*CommitLimit - *./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 135, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Commited", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 191, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_file - File-backed memory on active LRU list", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Active / Inactive Detail", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 130, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Writeback - Memory which is actively being written back to disk", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Writeback and Dirty", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 138, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Mapped - Used memory in mapped pages files which have been mmaped, such as libraries", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemPmdMapped - Ammount of shared (shmem/tmpfs) memory backed by huge pages", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Shared and Mapped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 131, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Slab", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 68 - }, - "id": 70, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocChunk - Largest contigious block of vmalloc area which is free", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocTotal - Total size of vmalloc memory area", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Vmalloc", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 68 - }, - "id": 159, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Bounce - Memory used for block device bounce buffers", - "refId": "A", - "step": 240 - } - ], - "title": "Memory Bounce", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Inactive *./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 78 - }, - "id": 129, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonHugePages - Memory in anonymous huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonPages - Memory in user pages not backed by files", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Anonymous", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 78 - }, - "id": 160, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Kernel / CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 88 - }, - "id": 140, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", - "refId": "C", - "step": 240 - } - ], - "title": "Memory HugePages Counter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 88 - }, - "id": 71, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages - Total size of the pool of huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Hugepagesize - Huge Page size", - "refId": "B", - "step": 240 - } - ], - "title": "Memory HugePages Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 98 - }, - "id": 128, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "DirectMap1G - Amount of pages mapped as this size", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap2M - Amount of pages mapped as this size", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap4K - Amount of pages mapped as this size", - "refId": "C", - "step": 240 - } - ], - "title": "Memory DirectMap", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 98 - }, - "id": 137, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Unevictable and MLocked", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 108 - }, - "id": 132, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage", - "refId": "A", - "step": 240 - } - ], - "title": "Memory NFS", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Memory Meminfo", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 267, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*out/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 176, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pagesin - Page in operations", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pagesout - Page out operations", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Pages In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*out/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 22, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pswpin - Pages swapped in", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pswpout - Pages swapped out", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Pages Swap In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "faults", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pgfault - Page major and minor fault operations" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 175, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgfault - Page major and minor fault operations", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgmajfault - Major page fault operations", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgminfault - Minor page fault operations", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Page Faults", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 307, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "oom killer invocations ", - "refId": "A", - "step": 240 - } - ], - "title": "OOM Killer", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Memory Vmstat", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 293, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Variation*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 260, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Estimated error in seconds", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Time offset in between local system and reference clock", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum error in seconds", - "refId": "C", - "step": 240 - } - ], - "title": "Time Syncronized Drift", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 291, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Phase-locked loop time adjust", - "refId": "A", - "step": 240 - } - ], - "title": "Time PLL Adjust", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Variation*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 50 - }, - "id": 168, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Local clock frequency adjustment", - "refId": "B", - "step": 240 - } - ], - "title": "Time Syncronized Status", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 50 - }, - "id": 294, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Seconds between clock ticks", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "International Atomic Time (TAI) offset", - "refId": "B", - "step": 240 - } - ], - "title": "Time Misc", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "System Timesync", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 312, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 62, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Processes blocked waiting for I/O to complete", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Processes in runnable state", - "refId": "B", - "step": 240 - } - ], - "title": "Processes Status", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 315, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ state }}", - "refId": "A", - "step": 240 - } - ], - "title": "Processes State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "forks / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 148, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Processes forks second", - "refId": "A", - "step": 240 - } - ], - "title": "Processes Forks", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max.*/" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 149, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Processes virtual memory size in bytes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum amount of virtual memory available in bytes", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Processes virtual memory size in bytes", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum amount of virtual memory available in bytes", - "refId": "D", - "step": 240 - } - ], - "title": "Processes Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "PIDs limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 313, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Number of PIDs", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PIDs limit", - "refId": "B", - "step": 240 - } - ], - "title": "PIDs Number and Limit", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*waiting.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 305, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }} - seconds spent running a process", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", - "refId": "B", - "step": 240 - } - ], - "title": "Process schedule stats Running / Waiting", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Threads limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 57 - }, - "id": 314, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Allocated threads", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Threads limit", - "refId": "B", - "step": 240 - } - ], - "title": "Threads Number and Limit", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "System Processes", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 269, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 42 - }, - "id": 8, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Context switches", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Interrupts", - "refId": "B", - "step": 240 - } - ], - "title": "Context Switches / Interrupts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 42 - }, - "id": 7, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_load1{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Load 1m", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_load5{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Load 5m", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_load15{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Load 15m", - "refId": "C", - "step": 240 - } - ], - "title": "System Load", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Critical*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 52 - }, - "id": 259, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ type }} - {{ info }}", - "refId": "A", - "step": 240 - } - ], - "title": "Interrupts Detail", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 52 - }, - "id": 306, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }}", - "refId": "A", - "step": 240 - } - ], - "title": "Schedule timeslices executed by each cpu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 62 - }, - "id": 151, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Entropy available to random number generators", - "refId": "A", - "step": 240 - } - ], - "title": "Entropy", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 62 - }, - "id": 308, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Time spent", - "refId": "A", - "step": 240 - } - ], - "title": "CPU time spent in user and system contexts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 64, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum open file descriptors", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Open file descriptors", - "refId": "B", - "step": 240 - } - ], - "title": "File Descriptors", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "System Misc", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 304, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "temperature", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "celsius" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Critical*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 158, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} temp", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Critical", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Critical Historical", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Max", - "refId": "E", - "step": 240 - } - ], - "title": "Hardware temperature monitor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 43 - }, - "id": 300, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Current {{ name }} in {{ type }}", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Max {{ name }} in {{ type }}", - "refId": "B", - "step": 240 - } - ], - "title": "Throttle cooling device", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 302, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ power_supply }} online", - "refId": "A", - "step": 240 - } - ], - "title": "Power supply", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Hardware Misc", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 296, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 297, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ name }} Connections", - "refId": "A", - "step": 240 - } - ], - "title": "Systemd Sockets", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF9830", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#73BF69", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Deactivating" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFCB7D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Activating" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C8F2C2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 298, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Activating", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Active", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Deactivating", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Failed", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inactive", - "refId": "E", - "step": 240 - } - ], - "title": "Systemd Units State", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Systemd", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 270, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The number (after merges) of I/O requests completed per second for the device", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 9, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{device}} - Reads completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Writes completed", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps Completed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The number of bytes read from or written to the device per second", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 33, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "{{device}} - Read bytes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Written bytes", - "refId": "B", - "step": 240 - } - ], - "title": "Disk R/W Data", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "time. read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 37, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - Read wait time avg", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Write wait time avg", - "refId": "B", - "step": 240 - } - ], - "title": "Disk Average Wait Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The average queue length of the requests that were issued to the device", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "aqu-sz", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 35, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}}", - "refId": "A", - "step": 240 - } - ], - "title": "Average Queue Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The number of read and write requests merged per second that were queued to the device", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "I/Os", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 133, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Read merged", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Write merged", - "refId": "B", - "step": 240 - } - ], - "title": "Disk R/W Merged", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "%util", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 51 - }, - "id": 36, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - IO", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - discard", - "refId": "B", - "step": 240 - } - ], - "title": "Time Spent Doing I/Os", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Outstanding req.", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 61 - }, - "id": 34, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - IO now", - "refId": "A", - "step": 240 - } - ], - "title": "Instantaneous Queue Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IOs", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 61 - }, - "id": 301, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - Discards completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Discards merged", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps Discards completed / merged", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Storage Disk", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 271, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 43, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Available", - "metric": "", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Free", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Size", - "refId": "C", - "step": 240 - } - ], - "title": "Filesystem space available", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "file nodes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 41, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Free file nodes", - "refId": "A", - "step": 240 - } - ], - "title": "File Nodes Free", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "files", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 28, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Max open files", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Open files", - "refId": "B", - "step": 240 - } - ], - "title": "File Descriptor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "file Nodes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 219, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - File nodes total", - "refId": "A", - "step": 240 - } - ], - "title": "File Nodes Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "/ ReadOnly" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 44, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - ReadOnly", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Device error", - "refId": "B", - "step": 240 - } - ], - "title": "Filesystem in ReadOnly / Error", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Storage Filesystem", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 272, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 60, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic by Packets", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 142, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive errors", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Rransmit errors", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 143, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive drop", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit drop", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Drop", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 43 - }, - "id": 141, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive compressed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit compressed", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Compressed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 146, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive multicast", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Multicast", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 144, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive fifo", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit fifo", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Fifo", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 145, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive frame", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Frame", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 231, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Statistic transmit_carrier", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Carrier", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 73 - }, - "id": 232, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit colls", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Colls", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "entries", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "NF conntrack limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 73 - }, - "id": 61, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "NF conntrack entries", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "NF conntrack limit", - "refId": "B", - "step": 240 - } - ], - "title": "NF Contrack", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Entries", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 230, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - ARP entries", - "refId": "A", - "step": 240 - } - ], - "title": "ARP Entries", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 288, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - Bytes", - "refId": "A", - "step": 240 - } - ], - "title": "MTU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 93 - }, - "id": 280, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - Speed", - "refId": "A", - "step": 240 - } - ], - "title": "Speed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 93 - }, - "id": 289, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - Interface transmit queue length", - "refId": "A", - "step": 240 - } - ], - "title": "Queue Length", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packetes drop (-) / process (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Dropped.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 103 - }, - "id": 290, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Processed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Dropped", - "refId": "B", - "step": 240 - } - ], - "title": "Softnet Packets", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 103 - }, - "id": 310, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Squeezed", - "refId": "A", - "step": 240 - } - ], - "title": "Softnet Out of Quota", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 113 - }, - "id": 309, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{interface}} - Operational state UP", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "instant": false, - "legendFormat": "{{device}} - Physical link state", - "refId": "B" - } - ], - "title": "Network Operational Status", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Network Traffic", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 273, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 63, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_alloc - Allocated sockets", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_inuse - Tcp sockets currently in use", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_mem - Used memory for tcp", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_orphan - Orphan sockets", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_tw - Sockets wating close", - "refId": "E", - "step": 240 - } - ], - "title": "Sockstat TCP", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 124, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "UDP_inuse - Udp sockets currently in use", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "UDP_mem - Used memory for udp", - "refId": "C", - "step": 240 - } - ], - "title": "Sockstat UDP", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 125, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "FRAG_inuse - Frag sockets currently in use", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RAW_inuse - Raw sockets currently in use", - "refId": "C", - "step": 240 - } - ], - "title": "Sockstat FRAG / RAW", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 220, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "mem_bytes - TCP sockets in that state", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "mem_bytes - UDP sockets in that state", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "FRAG_memory - Used memory for frag", - "refId": "C" - } - ], - "title": "Sockstat Memory Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "sockets", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 68 - }, - "id": 126, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Sockets_used - Sockets currently in use", - "refId": "A", - "step": 240 - } - ], - "title": "Sockstat Used", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Network Sockstat", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 274, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "octects out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 221, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InOctets - Received octets", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "OutOctets - Sent octets", - "refId": "B", - "step": 240 - } - ], - "title": "Netstat IP In / Out Octets", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 81, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Forwarding - IP forwarding", - "refId": "A", - "step": 240 - } - ], - "title": "Netstat IP Forwarding", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 59 - }, - "id": 115, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", - "refId": "B", - "step": 240 - } - ], - "title": "ICMP In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 59 - }, - "id": 50, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", - "refId": "A", - "step": 240 - } - ], - "title": "ICMP Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Snd.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 69 - }, - "id": 55, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InDatagrams - Datagrams received", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "OutDatagrams - Datagrams sent", - "refId": "B", - "step": 240 - } - ], - "title": "UDP In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 69 - }, - "id": 109, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RcvbufErrors - UDP buffer errors received", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "SndbufErrors - UDP buffer errors send", - "refId": "E", - "step": 240 - } - ], - "title": "UDP Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Snd.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 79 - }, - "id": 299, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", - "refId": "B", - "step": 240 - } - ], - "title": "TCP In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 79 - }, - "id": 104, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "OutRsts - Segments sent with RST flag", - "refId": "F" - } - ], - "title": "TCP Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "connections", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*MaxConn *./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 89 - }, - "id": 85, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")", - "refId": "B", - "step": 240 - } - ], - "title": "TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Sent.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 89 - }, - "id": 91, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "SyncookiesRecv - SYN cookies received", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "SyncookiesSent - SYN cookies sent", - "refId": "C", - "step": 240 - } - ], - "title": "TCP SynCookie", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "connections", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 99 - }, - "id": 82, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", - "refId": "B", - "step": 240 - } - ], - "title": "TCP Direct Transition", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Network Netstat", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 279, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 50 - }, - "id": 40, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape duration", - "refId": "A", - "step": 240 - } - ], - "title": "Node Exporter Scrape Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*error.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 50 - }, - "id": 157, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape success", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", - "refId": "B", - "step": 240 - } - ], - "title": "Node Exporter Scrape", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Node Exporter", - "type": "row" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [ - "linux" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Job", - "multi": false, - "name": "job", - "options": [], - "query": { - "query": "label_values(node_uname_info, job)", - "refId": "Prometheus-job-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", - "hide": 0, - "includeAll": false, - "label": "Host:", - "multi": false, - "name": "node", - "options": [], - "query": { - "query": "label_values(node_uname_info{job=\"$job\"}, instance)", - "refId": "Prometheus-node-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" - }, - "hide": 2, - "includeAll": false, - "multi": false, - "name": "diskdevices", - "options": [ - { - "selected": true, - "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" - } - ], - "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Node Exporter Full", - "uid": "rYdddlPWk", - "version": 9, - "weekStart": "" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-cluster_rev1.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-cluster_rev1.json deleted file mode 100644 index d9eda68..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-cluster_rev1.json +++ /dev/null @@ -1,971 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.1.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "natel-discrete-panel", - "name": "Discrete", - "version": "0.0.8-pre" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": 6278, - "graphTooltip": 1, - "id": null, - "iteration": 1527401341975, - "links": [], - "panels": [ - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "title": "Allocations", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "${DS_PROMETHEUS}", - "format": "percentunit", - "gauge": { - "maxValue": 1, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 0, - "y": 1 - }, - "id": 33, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "host", - "repeatDirection": "v", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "CPU", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "${DS_PROMETHEUS}", - "format": "percentunit", - "gauge": { - "maxValue": 1, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 5, - "y": 1 - }, - "id": 40, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "host", - "repeatDirection": "v", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Memory", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 2, - "format": "percentunit", - "gauge": { - "maxValue": 1, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 10, - "y": 1 - }, - "id": 48, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "host", - "repeatDirection": "v", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "nomad_client_unallocated_disk{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_disk{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_disk{datacenter=~\"$datacenter\", host=~\"$host\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Disk", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "backgroundColor": "rgba(128,128,128,0.1)", - "colorMaps": [ - { - "color": "#CCC", - "text": "N/A" - } - ], - "crosshairColor": "#8F070C", - "datasource": "${DS_PROMETHEUS}", - "display": "timeline", - "expandFromQueryS": 0, - "extendLastValue": true, - "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 1 - }, - "highlightOnMouseover": false, - "id": 58, - "legendSortBy": "-ms", - "lineColor": "rgba(0,0,0,0.1)", - "links": [], - "metricNameColor": "#000000", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "host", - "repeatDirection": "v", - "rowHeight": 24, - "showDistinctCount": false, - "showLegend": false, - "showLegendNames": false, - "showLegendPercent": true, - "showLegendValues": false, - "showTimeAxis": true, - "showTransitionCount": false, - "targets": [ - { - "expr": "nomad_client_allocations_migrating{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Migrating", - "refId": "A" - }, - { - "expr": "nomad_client_allocations_blocked{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Blocked", - "refId": "B" - }, - { - "expr": "nomad_client_allocations_pending{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pending", - "refId": "C" - }, - { - "expr": "nomad_client_allocations_running{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Running", - "refId": "D" - }, - { - "expr": "nomad_client_allocations_terminal{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Terminal", - "refId": "E" - } - ], - "textSize": 14, - "textSizeTime": 12, - "timeTextColor": "#d8d9da", - "title": "Summary", - "type": "natel-discrete-panel", - "units": "short", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueTextColor": "#000000", - "writeAllValues": false, - "writeLastValue": true, - "writeMetricNames": true - }, - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 2, - "title": "Host Status", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 1, - "format": "dtdurations", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 7 - }, - "id": 4, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "host", - "repeatDirection": "v", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "nomad_client_uptime{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Uptime($host)", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#cca300", - "#508642", - "#508642" - ], - "datasource": "${DS_PROMETHEUS}", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 4, - "y": 7 - }, - "id": 7, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "host", - "repeatDirection": "v", - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "nomad_client_host_cpu_idle{datacenter=\"$datacenter\", host=\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "20,80", - "title": "CPU Idle($host)", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 5, - "w": 5, - "x": 9, - "y": 7 - }, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "nomad_client_host_memory_total{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Total", - "refId": "B" - }, - { - "expr": "nomad_client_host_memory_free{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "Free", - "refId": "A" - }, - { - "expr": "nomad_client_host_memory_used{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "C" - }, - { - "expr": "nomad_client_host_memory_available{datacenter=~\"$datacenter\", host=~\"$host\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Available", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory ($host)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 5, - "w": 5, - "x": 14, - "y": 7 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(nomad_client_host_disk_size{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Total - {{disk}}", - "refId": "B" - }, - { - "expr": "avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Available - {{disk}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Disk Usage ($host)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 5, - "w": 5, - "x": 19, - "y": 7 - }, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(nomad_client_host_disk_inodes_percent{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{disk}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Disk Inodes ($host)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "1m", - "schemaVersion": 16, - "style": "dark", - "tags": [ - "nomad", - "host", - "cluster" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": false, - "label": "DC", - "multi": false, - "name": "datacenter", - "options": [], - "query": "label_values(nomad_client_uptime, datacenter)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "Host", - "multi": true, - "name": "host", - "options": [], - "query": "label_values(nomad_client_uptime{datacenter=~\"$datacenter\"}, host)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Nomad Cluster", - "uid": "CiP3mZVik", - "version": 26, - "description": "Nomad built-in telemetry https://github.com/dereknex/nomad-grafana" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-jobs_rev1.json b/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-jobs_rev1.json deleted file mode 100644 index 8cf916d..0000000 --- a/ansible/setup/stack_observability/templates/grafana/provisioning/dashboards/nomad-jobs_rev1.json +++ /dev/null @@ -1,486 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.1.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": 6281, - "graphTooltip": 0, - "id": null, - "iteration": 1527401878265, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(nomad_client_allocs_cpu_total_percent{host=~\"$host\"}) by(exported_job, task)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{task}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage Percent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 3, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(nomad_client_allocs_cpu_total_ticks{host=~\"$host\"}) by(exported_job, task)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{task}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU Total Ticks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 3, - "format": "timeticks", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(nomad_client_allocs_memory_rss{host=~\"$host\"}) by(exported_job, task)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{task}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "RSS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 3, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "host", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(nomad_client_allocs_memory_cache{host=~\"$host\"}) by(exported_job, task)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{task}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory Cache", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 3, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": false, - "label": "DC", - "multi": false, - "name": "datacenter", - "options": [], - "query": "label_values(nomad_client_uptime, datacenter)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": "Host", - "multi": true, - "name": "host", - "options": [], - "query": "label_values(nomad_client_uptime{datacenter=~\"$datacenter\"}, host)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Nomad Jobs", - "uid": "TvqbbhViz", - "version": 12, - "description": "Nomad Jobs metrics" -} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/AllocRestartingPermanantly.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/AllocRestartingPermanantly.json new file mode 100644 index 0000000..1ef62ce --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/AllocRestartingPermanantly.json @@ -0,0 +1 @@ +{"annotations":{"description":"Allocation '{{ $labels.exported_job}}' '{{ $labels.task_group}}' '{{ $labels.task}}' '{{ $labels.instance}}' is restarting permanantly","summary":"Allocation '{{ $labels.exported_job}}' '{{ $labels.task_group}}' '{{ $labels.task}}' '{{ $labels.instance}}' is restarting permanantly"},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"sum by (exported_job,task_group,task,instance) (sum_over_time(nomad_client_allocs_restart[1m]))","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[1],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"5m","id":60,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"AllocRestartingPermanantly","uid":"f0296e30-8b47-497b-8982-ab4d8709edaa","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBockExporterCheck.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBockExporterCheck.json new file mode 100644 index 0000000..5a2dafb --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBockExporterCheck.json @@ -0,0 +1 @@ +{"annotations":{"__dashboardUid__":"bd759518-5047-4f7e-9b7b-e30081fbeb1a","__panelId__":"2","summary":"Missing scrapes from blackbox exporter"},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"absent_over_time({job=\"integrations/blackbox-checks\"}[1m])","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":53,"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"BlackBockExporterCheck","uid":"cc234998-4dfb-44c0-a1b5-1cb1f7712eae","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBoxChecks.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBoxChecks.json new file mode 100644 index 0000000..d82c084 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/BlackBoxChecks.json @@ -0,0 +1 @@ +{"annotations":{"__dashboardUid__":"bd759518-5047-4f7e-9b7b-e30081fbeb1a","__panelId__":"1","description":"Blackbox check for '{{ $labels.instance}}' failed for 1m within last 5m","summary":"Blackbox check for '{{ $labels.instance}}' failed for 1m within last 5m.\nEither the agent is not running on host '{{ $labels.instance}}' or the scrape targets are not configured as expected"},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"count by (job,instance) (probe_success{job=\"integrations/blackbox-checks\"}==0) or count by (job,instance) (probe_failed_due_to_regex{job=\"integrations/blackbox-checks\"}==1)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":54,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"BlackBoxChecks","uid":"ab59d12a-e9bf-4efa-8ae6-fb8942c1ce60","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Service Down.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Service Down.json new file mode 100644 index 0000000..0804cd2 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Service Down.json @@ -0,0 +1 @@ +{"annotations":{"description":"Consul service is down since 1m for job '{{ $labels.instance}}' '{{ $labels.service_name}}' the last 5m","summary":"Values of 1 mean that all nodes for the service are passing. Values of 0 mean at least one node for the service is not passing."},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"min(consul_catalog_service_node_healthy) by (service_name)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[1],"type":"lt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":57,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"Consul Service Down","uid":"bd82f500-7150-4358-995a-7bfcf959a751","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Up metric.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Up metric.json new file mode 100644 index 0000000..62fad8c --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul Up metric.json @@ -0,0 +1 @@ +{"annotations":{"__dashboardUid__":"bd759518-5047-4f7e-9b7b-e30081fbeb1a","__panelId__":"4","description":"Consul Up metric is 0 since 1m for job '{{ $labels.instance}}' '{{ $labels.job}}' '{{ $labels.service}}' the last 5m","summary":"Consul Up metric is 0 since 1m for job '{{ $labels.instance}}' '{{ $labels.job}}' '{{ $labels.service}}' the last 5m"},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"sum_over_time (consul_up{}[1m]) ","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[1],"type":"lt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":56,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"Consul Up metric","uid":"a9151b86-3927-4ded-a916-14cceac5a16c","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul healthcheck critical.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul healthcheck critical.json new file mode 100644 index 0000000..1a8daaa --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul healthcheck critical.json @@ -0,0 +1 @@ +{"annotations":{"description":"Consul healthcheck failed since 1m for service'{{ $labels.service_name}}' on node '{{ $labels.node}}'","summary":"Consul healthcheck failed since 1m for service'{{ $labels.service_name}}' on node '{{ $labels.node}}'"},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"count( consul_health_service_status{status=\"critical\"} ==1) by(service_name)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"2m","id":59,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"Consul healthcheck critical","uid":"f55fa344-54fb-486b-852e-6f336c5b46cb","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul service failing.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul service failing.json new file mode 100644 index 0000000..2de66ff --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Consul service failing.json @@ -0,0 +1 @@ +{"annotations":{"__dashboardUid__":"bd759518-5047-4f7e-9b7b-e30081fbeb1a","__panelId__":"4","description":"Consul Up metric is 0 since 1m for job '{{ $labels.instance}}' '{{ $labels.job}}' '{{ $labels.service}}' the last 5m","summary":"Consul Up metric is 0 since 1m for job '{{ $labels.instance}}' '{{ $labels.job}}' '{{ $labels.service}}' the last 5m"},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"min(consul_catalog_service_node_healthy) by (service_name)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[1],"type":"lt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":58,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"Consul service failing","uid":"d4cbdbef-1c43-4843-bd79-bae6d0fdc56f","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Up metrics.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Up metrics.json new file mode 100644 index 0000000..8d6ce62 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/BackBoneServices/Up metrics.json @@ -0,0 +1 @@ +{"annotations":{"__dashboardUid__":"bd759518-5047-4f7e-9b7b-e30081fbeb1a","__panelId__":"3","description":"Up metric is 0 since for job '{{ $labels.instance}}' '{{ $labels.job}}' '{{ $labels.service}}' for 1m the last 5m","summary":"Up metric is 0 since for job '{{ $labels.instance}}' '{{ $labels.job}}' '{{ $labels.service}}' for 1m the last 5m .\n\nThe scrape can fail when: \n\nWhen scrape target was unreachable during the scrape.\n\nWhen the target didn't return response during the configured timeout. The timeout can be configured via scrape_timeout option. By default it is set to 10 seconds. \n\nWhen there was a network issue during the scrape, which prevented from successful scrape.\n\nWhen the scrape target returns incorrect or incomplete response. The response must contain metrics with values in Prometheus text exposition format."},"condition":"Value","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"sum_over_time (up{}[1m]) ","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"probe_not_successORprobe_failed_due_to_regex"},"refId":"probe_not_successORprobe_failed_due_to_regex","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"probe_not_successORprobe_failed_due_to_regex","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"Last","type":"reduce"},"refId":"Last","relativeTimeRange":{"from":300}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[1],"type":"lt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"Last","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"Value","type":"threshold"},"refId":"Value","relativeTimeRange":{"from":300}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":55,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"BackBoneServices","title":"Up metrics","uid":"fda9ac05-464a-4496-a57d-f4d9683038f9","updated":"2023-05-24T08:43:21Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/Failed Image pulls last hour.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/Failed Image pulls last hour.json new file mode 100644 index 0000000..d28eab4 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/Failed Image pulls last hour.json @@ -0,0 +1 @@ +{"annotations":{"Message":"Failed to pull container image for {{ $labels.job }} within last hor","__dashboardUid__":"fb44366b-6148-455a-baef-eb306e333125","__panelId__":"2"},"condition":"C","data":[{"datasourceUid":"loki","model":{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"count_over_time({ingress=\"ingress.logs.journald\"} |= \"Failed to pull\"[1m])","intervalMs":1000,"legendFormat":"\"High error rate in for job {{ job }} with log level {{ level }}\"","maxDataPoints":43200,"queryType":"range","refId":"A"},"queryType":"range","refId":"A","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0.5],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":3600}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"5m","id":61,"noDataState":"OK","orgID":1,"ruleGroup":"LogAnalysis","title":"Failed Image pulls last hour","uid":"c53196d1-fd38-44f3-836a-ed8b74b994d0","updated":"2023-05-24T14:15:23Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/High Log Error Rate Last Hour.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/High Log Error Rate Last Hour.json new file mode 100644 index 0000000..42b83bf --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/LogAnalysis/High Log Error Rate Last Hour.json @@ -0,0 +1 @@ +{"annotations":{"Message":"High error rate in for job {{ $labels.job }} with log level {{ $labels.level }}","__dashboardUid__":"nQbXGkP4z","__panelId__":"2"},"condition":"C","data":[{"datasourceUid":"loki","model":{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":" sum(rate({ingress=\"ingress.logs.journald\",job=~\".*\"} |~ \"error|failed|missing\" !~ `errors=0|failed=false` !~ `\"logfmt_failedExemplarsCount\":.*\"0\"` !~ \"context canceled\"[5m])) by (job,level) /\r\n sum(rate({ingress=\"ingress.logs.journald\"}[5m])) by (job,level)\r\n ","intervalMs":1000,"legendFormat":"\"High error rate in for job {{ job }} with log level {{ level }}\"","maxDataPoints":43200,"queryType":"range","refId":"A"},"queryType":"range","refId":"A","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0.5],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":3600}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"15m","id":3,"noDataState":"Alerting","orgID":1,"ruleGroup":"LogAnalysis","title":"High Log Error Rate Last Hour","uid":"db7Nx4EVk","updated":"2023-05-24T14:15:23Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockNotSynchronising.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockNotSynchronising.json new file mode 100644 index 0000000..45c5727 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockNotSynchronising.json @@ -0,0 +1 @@ +{"annotations":{"description":"Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.","summary":"Clock not synchronising."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"min_over_time(node_timex_sync_status{job=\"integrations/node_exporter\"}[5m]) == 0 and node_timex_maxerror_seconds{job=\"integrations/node_exporter\"} \u003e= 16","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":1800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":1800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":1800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"10m","id":75,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeClockNotSynchronising","uid":"ce478b0d-8939-4dcc-ba98-8be5236b997e","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockSkewDetected.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockSkewDetected.json new file mode 100644 index 0000000..e894524 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeClockSkewDetected.json @@ -0,0 +1 @@ +{"annotations":{"description":"Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.","summary":"Clock skew detected."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_timex_offset_seconds{job=\"integrations/node_exporter\"} \u003e 0.05 and deriv(node_timex_offset_seconds{job=\"integrations/node_exporter\"}[5m]) \u003e= 0) or (node_timex_offset_seconds{job=\"integrations/node_exporter\"} \u003c -0.05 and deriv(node_timex_offset_seconds{job=\"integrations/node_exporter\"}[5m]) \u003c= 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":1800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":1800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":1800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"10m","id":74,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeClockSkewDetected","uid":"de94327f-5a15-4fda-b374-19316cb4c021","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit.json new file mode 100644 index 0000000..ee85a5d --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit.json @@ -0,0 +1 @@ +{"annotations":{"description":"File descriptors limit at {{ $labels.instance }} is currently at {{ printf \"%.2f\" $value }}%.","summary":"Kernel is predicted to exhaust file descriptors limit soon."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filefd_allocated{job=\"integrations/node_exporter\"} * 100 / node_filefd_maximum{job=\"integrations/node_exporter\"} \u003e 70)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":900}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":900}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":900}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":78,"labels":{"severity":"warning"},"noDataState":"NoData","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFileDescriptorLimit","uid":"c35d0abc-9493-404d-b584-7a9849c391c4","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit90pct.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit90pct.json new file mode 100644 index 0000000..9fe1fe2 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFileDescriptorLimit90pct.json @@ -0,0 +1 @@ +{"annotations":{"description":"File descriptors limit at {{ $labels.instance }} is currently at {{ printf \"%.2f\" $value }}%.","summary":"Kernel is predicted to exhaust file descriptors limit soon."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filefd_allocated{job=\"integrations/node_exporter\"} * 100 / node_filefd_maximum{job=\"integrations/node_exporter\"} \u003e 90)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":3600}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"15m","id":79,"labels":{"severity":"critical"},"noDataState":"NoData","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFileDescriptorLimit90pct","uid":"e5ce818d-4999-4748-ba1e-770b4caf4402","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles3pct.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles3pct.json new file mode 100644 index 0000000..ebf538c --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles3pct.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem has less than 3% inodes left.","summary":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filesystem_files_free{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_files{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 3 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":10800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":69,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemAlmostOutOfFiles3pct","uid":"b04dfece-8690-4520-b771-3e48d2791521","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles5pct.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles5pct.json new file mode 100644 index 0000000..a00cc19 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfFiles5pct.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem has less than 5% inodes left.","summary":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filesystem_files_free{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_files{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 5 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":10800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":68,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemAlmostOutOfFiles5pct","uid":"e149b653-ebde-4949-9c47-70bf6df568f6","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace10.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace10.json new file mode 100644 index 0000000..e9a7e98 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace10.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.","summary":"Filesystem has less than 10% space left."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":" (node_filesystem_avail_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 10 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":3600}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"30m","id":65,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemAlmostOutOfSpace10","uid":"d97a9a54-b6ad-4aa6-9f65-9573d6555df1","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace15.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace15.json new file mode 100644 index 0000000..37cf610 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemAlmostOutOfSpace15.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.","summary":"ilesystem has less than 15% space left."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":" (node_filesystem_avail_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 15 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":3600}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"30m","id":64,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemAlmostOutOfSpace15","uid":"c5aa56f0-61c6-4870-a3fa-df76453e1001","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp24h.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp24h.json new file mode 100644 index 0000000..185e3e0 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp24h.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem is predicted to run out of inodes within the next 24 hours.","summary":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filesystem_files_free{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_files{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 40 and predict_linear(node_filesystem_files_free{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"}[6h], 24 * 60 * 60) \u003c 0 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":86400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":86400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":86400}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":66,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemFilesFillingUp24h","uid":"ead577a2-0e72-4d2e-a5dd-4ce64478775d","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp4h.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp4h.json new file mode 100644 index 0000000..5e46941 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemFilesFillingUp4h.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem is predicted to run out of inodes within the next 4 hours.","summary":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up fast."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filesystem_files_free{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_files{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 20 and predict_linear(node_filesystem_files_free{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"}[6h], 4 * 60 * 60) \u003c 0 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":14400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":14400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":14400}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":67,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemFilesFillingUp4h","uid":"eb9362dd-3c92-4ad3-b74d-bb7cf2b4b69d","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext24h.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext24h.json new file mode 100644 index 0000000..d1e2fcf --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext24h.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up.","summary":"Filesystem is predicted to run out of space within the next 24 hours."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_filesystem_avail_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 40 and predict_linear(node_filesystem_avail_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"}[6h], 24 * 60 * 60) \u003c 0 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":86400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":86400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":86400}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":62,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemSpaceFillingUpNext24h","uid":"a706ce12-fb45-447d-a450-dfb99c51d9b0","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext4h.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext4h.json new file mode 100644 index 0000000..4afccae --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeFilesystemSpaceFillingUpNext4h.json @@ -0,0 +1 @@ +{"annotations":{"description":"Filesystem is predicted to run out of space within the next 4 hours.","summary":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up fast."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":" (node_filesystem_avail_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} * 100 \u003c 20 and predict_linear(node_filesystem_avail_bytes{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"}[6h], 4 * 60 * 60) \u003c 0 and node_filesystem_readonly{fstype!=\"\",job=\"integrations/node_exporter\",mountpoint!=\"\"} == 0)","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":14400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":14400}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":14400}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":63,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeFilesystemSpaceFillingUpNext4h","uid":"f885c6ef-e6c8-4832-a3f7-ec0b27837df0","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeHighNumberConntrackEntriesUsed.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeHighNumberConntrackEntriesUsed.json new file mode 100644 index 0000000..54766e2 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeHighNumberConntrackEntriesUsed.json @@ -0,0 +1 @@ +{"annotations":{"description":"'{{ $value | humanizePercentage }} of conntrack entries are used.'","summary":"Number of conntrack are getting close to the limit."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) \u003e 0.75","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":900}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":900}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":900}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":72,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeHighNumberConntrackEntriesUsed","uid":"b75ff681-1caa-40ea-b93b-c55b2e9def2e","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkReceiveErrs.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkReceiveErrs.json new file mode 100644 index 0000000..32ee296 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkReceiveErrs.json @@ -0,0 +1 @@ +{"annotations":{"description":"'{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.'","summary":"Network interface is reporting many receive errors."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) \u003e 0.01","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":10800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":70,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeNetworkReceiveErrs","uid":"a347e813-0dd5-4c33-af8e-1d4a31fdd733","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkTransmitErrs.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkTransmitErrs.json new file mode 100644 index 0000000..bb2a8b2 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeNetworkTransmitErrs.json @@ -0,0 +1 @@ +{"annotations":{"description":"Network interface is reporting many transmit errors.","summary":"'{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.'"},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) \u003e 0.01","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":10800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":10800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1h","id":71,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeNetworkTransmitErrs","uid":"bc196df3-2baa-4163-a9fd-851689c0b330","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDegraded.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDegraded.json new file mode 100644 index 0000000..667fbf1 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDegraded.json @@ -0,0 +1 @@ +{"annotations":{"description":"RAID array '{{ $labels.device }}' on {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.","summary":"RAID Array is degraded"},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"node_md_disks_required{device!=\"\",job=\"integrations/node_exporter\"} - ignoring (state) (node_md_disks{device!=\"\",job=\"integrations/node_exporter\",state=\"active\"}) \u003e 0","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":3600}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":3600}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"15m","id":76,"labels":{"severity":"critical"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeRAIDDegraded","uid":"d77715d2-d0dd-4004-8d48-ff1fa5bafb41","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDiskFailure.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDiskFailure.json new file mode 100644 index 0000000..bc00d63 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeRAIDDiskFailure.json @@ -0,0 +1 @@ +{"annotations":{"description":"At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.","summary":"Failed device in RAID array"},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"node_md_disks{device!=\"\",job=\"integrations/node_exporter\",state=\"failed\"} \u003e 0","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":1800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":1800}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":1800}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"1m","id":77,"labels":{"severity":"warning"},"noDataState":"NoData","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeRAIDDiskFailure","uid":"b23986d7-8279-4ea7-932e-abc63e0d19aa","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeTextFileCollectorScrapeError.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeTextFileCollectorScrapeError.json new file mode 100644 index 0000000..075cb13 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/alert-rules/Alarms/NodeMetrics/NodeTextFileCollectorScrapeError.json @@ -0,0 +1 @@ +{"annotations":{"description":"Node Exporter text file collector failed to scrape.","summary":"Node Exporter text file collector failed to scrape."},"condition":"C","data":[{"datasourceUid":"mimir","model":{"editorMode":"code","expr":"node_textfile_scrape_error{job=\"integrations/node_exporter\"} == 1","hide":false,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":true,"refId":"A"},"refId":"A","relativeTimeRange":{"from":900}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[],"type":"gt"},"operator":{"type":"and"},"query":{"params":["B"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","type":"reduce"},"refId":"B","relativeTimeRange":{"from":900}},{"datasourceUid":"__expr__","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["C"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"B","hide":false,"intervalMs":1000,"maxDataPoints":43200,"refId":"C","type":"threshold"},"refId":"C","relativeTimeRange":{"from":900}}],"execErrState":"Error","folderUID":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b","for":"5m","id":73,"labels":{"severity":"warning"},"noDataState":"OK","orgID":1,"ruleGroup":"NodeMetrics","title":"NodeTextFileCollectorScrapeError","uid":"daa9c055-9625-49ca-898d-f80fbe80c670","updated":"2023-05-26T21:14:44Z"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/BlackBoxChecks.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/BlackBoxChecks.json new file mode 100644 index 0000000..47cb005 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/BlackBoxChecks.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":57,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":13,"w":12,"x":0,"y":0},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"absent_over_time({job=\"integrations/blackbox-checks\"}[1m])","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Availability integrations/blackbox-checks","transparent":true,"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":12,"w":12,"x":12,"y":0},"id":1,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"count by (job,instance) (probe_success{job=\"integrations/blackbox-checks\"}==0)","legendFormat":"Failed Probe {{instance}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":" count by (job,instance) (probe_failed_due_to_regex{job=\"integrations/blackbox-checks\"}==1)","hide":false,"legendFormat":"Failed Regex {{instance}}","range":true,"refId":"B"}],"title":"BlackBoxChecks","transparent":true,"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":11,"w":12,"x":12,"y":12},"id":4,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":" sum_over_time (consul_up{}[1m]) ","legendFormat":"{{instance}} {{job}} {{service}}","range":true,"refId":"A"}],"title":"Consul Up Metrics","transparent":true,"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":13},"id":3,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":" sum_over_time (up{}[1m]) ","legendFormat":"{{instance}} {{job}} {{service}}","range":true,"refId":"A"}],"title":"Missing Up Metrics","transparent":true,"type":"timeseries"}],"refresh":"","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-3h","to":"now"},"timepicker":{},"timezone":"","title":"BlackBoxChecks","uid":"bd759518-5047-4f7e-9b7b-e30081fbeb1a","version":6,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/Failed to pull images.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/Failed to pull images.json new file mode 100644 index 0000000..8225903 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/Failed to pull images.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":58,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"loki","uid":"loki"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":11,"w":24,"x":0,"y":0},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"count_over_time({ingress=\"ingress.logs.journald\"} |= \"Failed to pull\"[1m])","queryType":"range","refId":"A"}],"title":"Count failed image pulls ","transparent":true,"type":"timeseries"},{"datasource":{"type":"loki","uid":"loki"},"gridPos":{"h":20,"w":24,"x":0,"y":11},"id":1,"options":{"dedupStrategy":"none","enableLogDetails":true,"prettifyLogMessage":false,"showCommonLabels":false,"showLabels":false,"showTime":false,"sortOrder":"Descending","wrapLogMessage":false},"targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"{ingress=\"ingress.logs.journald\"} |= `Failed to pull` | json | line_format `[{{.job}} 💻 {{.host_name}}] [{{.log_levelEmoji}}] {{.message}} {{if .trace_trace_id}} traceID={{.trace_trace_id}} {{end}}`","key":"Q-cff41bc9-2feb-4454-ba7f-b77cc4757a3c-0","queryType":"range","refId":"A"}],"title":"Failed Pull Images","transparent":true,"type":"logs"}],"refresh":"","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-6h","to":"now"},"timepicker":{},"timezone":"","title":"Failed to pull images","uid":"fb44366b-6148-455a-baef-eb306e333125","version":2,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/LogAlerts.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/LogAlerts.json new file mode 100644 index 0000000..04feed3 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Alarms/LogAlerts.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":11,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":18,"w":11,"x":0,"y":0},"id":4,"options":{"alertInstanceLabelFilter":"{job=~\"$job\"}","alertName":"","dashboardAlerts":false,"datasource":"-- Grafana --","folder":{"title":"Alarms","uid":"c8670be5-3ca6-4897-ae95-6f9fd22efd8b"},"groupBy":["alertname"],"groupMode":"default","maxItems":200,"sortOrder":1,"stateFilter":{"error":true,"firing":true,"noData":false,"normal":false,"pending":true},"viewMode":"list"},"title":"Panel Title","transparent":true,"type":"alertlist"},{"datasource":{"type":"loki","uid":"loki"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"hue","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byRegexp","options":"Log count .*"},"properties":[{"id":"custom.drawStyle","value":"bars"},{"id":"custom.stacking","value":{"group":"B","mode":"normal"}},{"id":"custom.axisColorMode","value":"series"},{"id":"color","value":{"mode":"continuous-YlBl"}}]}]},"gridPos":{"h":18,"w":13,"x":11,"y":0},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":" sum(rate({ingress=\"ingress.logs.journald\",job=~\"${job}\"} |~ \"error|failed|missing\" !~ `errors=0|failed=false` !~ `\"logfmt_failedExemplarsCount\":.*\"0\"` [5m])) by (job,level) /\r\n sum(rate({ingress=\"ingress.logs.journald\"}[5m])) by (job,level)\r\n ","legendFormat":"\"High error rate in for job {{ job }} with log level {{ level }}\"","queryType":"range","refId":"A"},{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"sum(count_over_time({ingress=\"ingress.logs.journald\" ,job=~\"${job}\"} [5m])) by( host_name)","hide":false,"legendFormat":"Log count {{host_name}}","queryType":"range","refId":"B"}],"title":"Panel Title","transparent":true,"type":"timeseries"},{"datasource":{"type":"loki","uid":"loki"},"gridPos":{"h":15,"w":24,"x":0,"y":18},"id":5,"options":{"dedupStrategy":"none","enableLogDetails":true,"prettifyLogMessage":true,"showCommonLabels":false,"showLabels":false,"showTime":true,"sortOrder":"Descending","wrapLogMessage":true},"targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"{ingress=\"ingress.logs.journald\", job=~\"${job}\"} |~ `error|failed|missing` !~ `errors=0|failed=false` !~ `\"logfmt_failedExemplarsCount\":.*\"0\"` !~ \"context canceled\" | json | line_format `[{{.job}} 💻 {{.host_name}}] [{{.log_levelEmoji}}] {{.message}} {{if .error_message}} error={{.error_message}} {{end}} {{if .trace_trace_id}} traceID={{.trace_trace_id}} {{end}}`\r\n ","legendFormat":"\"High error rate in for job {{ job }} with log level {{ level }}\"","queryType":"range","refId":"A"}],"title":"Panel Title","transparent":true,"type":"logs"}],"refresh":"5s","revision":1,"schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[{"datasource":{"type":"loki","uid":"loki"},"filters":[],"hide":0,"name":"query0","skipUrlSync":false,"type":"adhoc"},{"current":{"selected":false,"text":".*","value":".*"},"hide":0,"label":"job","name":"job","options":[{"selected":true,"text":".*","value":".*"}],"query":".*","skipUrlSync":false,"type":"textbox"}]},"time":{"from":"now-6h","to":"now"},"timepicker":{},"timezone":"","title":"LogAlerts","uid":"nQbXGkP4z","version":33,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Alerts Overview.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Alerts Overview.json new file mode 100644 index 0000000..af4787f --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Alerts Overview.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":32,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":9,"w":24,"x":0,"y":0},"id":2,"options":{"alertInstanceLabelFilter":"","alertName":"","dashboardAlerts":false,"groupBy":[],"groupMode":"default","maxItems":20,"sortOrder":1,"stateFilter":{"error":true,"firing":true,"noData":false,"normal":false,"pending":true},"viewMode":"list"},"title":"Panel Title","type":"alertlist"},{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":10,"w":24,"x":0,"y":9},"id":1,"options":{"alertmanager":"grafana","expandAll":true},"title":"Panel Title","type":"alertGroups"}],"refresh":"","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-6h","to":"now"},"timepicker":{},"timezone":"","title":"Alerts Overview","uid":"feb77514-202b-4c58-bce0-0090c50aa292","version":1,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/LogsTest.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/LogsTest.json new file mode 100644 index 0000000..1cafa6a --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/LogsTest.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":3,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":7,"panels":[],"title":"Log level distribution for job type $__interval","type":"row"},{"datasource":{"type":"loki","uid":"loki"},"description":"","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"error "},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"debug "},"properties":[{"id":"color","value":{"fixedColor":"dark-blue","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"fatal "},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"info "},"properties":[{"id":"color","value":{"fixedColor":"dark-green","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trace "},"properties":[{"id":"color","value":{"fixedColor":"dark-purple","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"unknown "},"properties":[{"id":"color","value":{"mode":"fixed"}}]},{"matcher":{"id":"byName","options":"warn "},"properties":[{"id":"color","value":{"fixedColor":"dark-orange","mode":"fixed"}}]}]},"gridPos":{"h":3,"w":12,"x":0,"y":1},"id":2,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.4.7","repeat":"job_types","repeatDirection":"h","targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"sum by(level) (count_over_time({ingress=\"$ingress\",job_type=~\"$job_types\"} [$__interval]))","key":"Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0","legendFormat":"{{level}} ","queryType":"range","refId":"A"}],"title":"$job_types ","transformations":[],"transparent":true,"type":"stat"},{"datasource":{"type":"loki","uid":"loki"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"error "},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"debug "},"properties":[{"id":"color","value":{"fixedColor":"dark-blue","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"fatal "},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"info "},"properties":[{"id":"color","value":{"fixedColor":"dark-green","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trace "},"properties":[{"id":"color","value":{"fixedColor":"dark-purple","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"unknown "},"properties":[{"id":"color","value":{"mode":"fixed"}}]},{"matcher":{"id":"byName","options":"warn "},"properties":[{"id":"color","value":{"fixedColor":"dark-orange","mode":"fixed"}}]}]},"gridPos":{"h":14,"w":5,"x":0,"y":4},"id":4,"options":{"displayLabels":["percent"],"legend":{"displayMode":"list","placement":"right","showLegend":false,"values":[]},"pieType":"donut","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"sum by(level) (count_over_time({ingress=\"$ingress\"} [1w]))","key":"Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0","legendFormat":"{{level}} ","queryType":"range","refId":"A"}],"title":"Log distrubution","transformations":[],"transparent":true,"type":"piechart"},{"datasource":{"type":"loki","uid":"loki"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"debug "},"properties":[{"id":"color","value":{"fixedColor":"dark-blue","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"error "},"properties":[{"id":"color","value":{"fixedColor":"semi-dark-red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"fatal "},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"info "},"properties":[{"id":"color","value":{"fixedColor":"dark-green","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trace"},"properties":[{"id":"color","value":{"fixedColor":"dark-purple","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"unknown "},"properties":[{"id":"color","value":{"fixedColor":"text","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"warn "},"properties":[{"id":"color","value":{"fixedColor":"dark-orange","mode":"fixed"}}]}]},"gridPos":{"h":13,"w":19,"x":5,"y":4},"id":3,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"sum by(level) (count_over_time({ingress=\"$ingress\"} [$__interval]))","key":"Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0","legendFormat":"{{level}} ","queryType":"range","refId":"A"}],"title":"Log distrubution in $__interval","transformations":[{"disabled":true,"id":"labelsToFields","options":{"mode":"columns","valueLabel":"level"}}],"transparent":true,"type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":18},"id":9,"panels":[{"datasource":{"type":"loki","uid":"loki"},"description":"","fieldConfig":{"defaults":{"color":{"fixedColor":"dark-orange","mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"color-text"},"filterable":true,"inspect":false},"mappings":[{"options":{"pattern":".*","result":{"color":"super-light-blue","index":0}},"type":"regex"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"level"},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"thresholds"}},{"id":"mappings","value":[{"options":{"debug":{"color":"dark-blue","index":3},"error":{"color":"semi-dark-red","index":0},"fatal":{"color":"dark-red","index":1},"info":{"color":"dark-green","index":2},"trace":{"color":"dark-purple","index":4},"unknown":{"color":"text","index":5},"warn":{"color":"dark-orange","index":6}},"type":"value"}]}]}]},"gridPos":{"h":10,"w":24,"x":0,"y":19},"id":5,"options":{"footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"frameIndex":0,"showHeader":true,"sortBy":[{"desc":true,"displayName":"Count (sum)"}]},"pluginVersion":"9.4.3","targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"sum by(job_type, stack,job,level) (count_over_time({ingress=\"$ingress\"} [1w] ))","key":"Q-50af83e8-1f38-4ca0-8785-df5409a12c33-0","legendFormat":"","queryType":"instant","refId":"A","resolution":1}],"title":" ","transformations":[{"id":"organize","options":{"excludeByName":{},"indexByName":{"Time":0,"Value #A":5,"job":3,"job_type":1,"level":4,"stack":2},"renameByName":{"Time":"","Value #A":"Count","job_type":"Job Type","level":""}}},{"id":"groupBy","options":{"fields":{"Count":{"aggregations":["sum"],"operation":"aggregate"},"Job Type":{"aggregations":[],"operation":"groupby"},"job":{"aggregations":[],"operation":"groupby"},"job_type":{"aggregations":[],"operation":"groupby"},"level":{"aggregations":[],"operation":"groupby"},"stack":{"aggregations":[],"operation":"groupby"}}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Job Type"}]}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"job"}]}},{"id":"sortBy","options":{"fields":{},"sort":[{"desc":true,"field":"Count (sum)"}]}}],"transparent":true,"type":"table"}],"title":"Row title","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":19},"id":15,"panels":[{"datasource":{"type":"loki","uid":"loki"},"gridPos":{"h":25,"w":24,"x":0,"y":20},"id":13,"options":{"dedupStrategy":"exact","enableLogDetails":true,"prettifyLogMessage":false,"showCommonLabels":false,"showLabels":false,"showTime":true,"sortOrder":"Descending","wrapLogMessage":false},"targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"builder","expr":"{ingress=\"$ingress\", job_type=~\"$job_types\"} |= `` | json | line_format `{{.job}}@{{.host_name}} [{{.log_levelEmoji}} {{.level}}] [{{.tags}}] {{.message}}`","key":"Q-eef4e925-e736-47be-9448-c35f4c6acf7d-0","queryType":"range","refId":"A"}],"title":"New Panel","transparent":true,"type":"logs"}],"title":"Row title","type":"row"}],"refresh":"1s","revision":1,"schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[{"datasource":{"type":"loki","uid":"loki"},"filters":[],"hide":0,"name":"Filters","skipUrlSync":false,"type":"adhoc"},{"current":{"selected":false,"text":"ingress.logs.journald","value":"ingress.logs.journald"},"datasource":{"type":"loki","uid":"loki"},"definition":"label_values(job_type)","hide":0,"includeAll":false,"label":"ingress","multi":false,"name":"ingress","options":[],"query":{"label":"ingress","refId":"LokiVariableQueryEditor-VariableQuery","stream":"","type":1},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{"selected":true,"text":["All"],"value":["$__all"]},"datasource":{"type":"loki","uid":"loki"},"definition":"label_values(job_type)","hide":0,"includeAll":true,"label":"job_types","multi":true,"name":"job_types","options":[],"query":{"label":"job_type","refId":"LokiVariableQueryEditor-VariableQuery","stream":"{ingress=\"$ingress\"}","type":1},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-24h","to":"now"},"timepicker":{"refresh_intervals":["1s","5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"","title":"LogsTest","uid":"KMg_v90Vz","version":6,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Loki2.0 Global Metrics.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Loki2.0 Global Metrics.json new file mode 100644 index 0000000..99a1d93 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Loki2.0 Global Metrics.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"$$hashKey":"object:7","builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Loki metrics via 2.0","editable":true,"fiscalYearStartMonth":0,"gnetId":13407,"graphTooltip":0,"id":4,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","gridPos":{"h":2,"w":24,"x":0,"y":0},"id":21,"interval":"","options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"\u003cdiv class=\"left dashboard-header\"\u003e\n \u003cimg src=\"https://marketplace-assets.digitalocean.com/logos/loki-logo.svg\" style=\"height:29px;\"/\u003e\n \u003cspan\u003eLoki Global Metrics\u003c/span\u003e\n\u003c/div\u003e","mode":"html"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"loki_build_info","format":"table","instant":false,"interval":"","legendFormat":"","refId":"A"}],"transparent":true,"type":"text"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":4,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":0,"y":2},"id":59,"interval":"","options":{"colorMode":"background","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"/^version$/","values":false},"textMode":"auto"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"loki_build_info","format":"table","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"Loki Version","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":3,"y":2},"id":10,"interval":"","options":{"colorMode":"value","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(log_messages_total)","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"Message Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":1}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":6,"y":2},"id":44,"interval":"","options":{"colorMode":"background","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"textMode":"value"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"cortex_prometheus_notifications_alertmanagers_discovered","format":"time_series","instant":true,"interval":"","legendFormat":"","refId":"A"}],"title":"Current Alerts","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":9,"y":2},"id":48,"interval":"","options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"textMode":"value"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"cortex_prometheus_notifications_sent_total","format":"time_series","instant":true,"interval":"","legendFormat":"","refId":"A"}],"title":"Alerts Event Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":4,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":12,"y":2},"id":23,"interval":"","options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(loki_store_series_total)","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"Store Series Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":4,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":15,"y":2},"id":41,"interval":"","options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(loki_ingester_chunk_stored_bytes_total)","instant":true,"interval":"","legendFormat":"","refId":"A"}],"title":"Store Chunks Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":18,"y":2},"id":49,"interval":"","options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"textMode":"value"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(cortex_prometheus_rule_group_rules)","format":"time_series","instant":true,"interval":"","legendFormat":"","refId":"A"}],"title":"Rules Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"decimals":4,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":21,"y":2},"id":24,"interval":"","options":{"colorMode":"value","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.3.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"loki_panic_total","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"Panic ","type":"stat"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":5,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":0,"y":5},"hiddenSeries":false,"id":8,"interval":"","legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(irate(log_messages_total[1m])) by (level)","interval":"","legendFormat":"{{ level }}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Messages Input","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"short","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":12,"y":5},"hiddenSeries":false,"id":53,"interval":"","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(cortex_s3_request_duration_seconds_bucket[5m])) by (le,operation))","interval":"","legendFormat":"{{operation}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"S3 Request Durations","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":0,"y":11},"hiddenSeries":false,"id":11,"interval":"","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket[5m])) by (le,route))","interval":"","legendFormat":"{{ route }}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"API Request Durations","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":12,"y":11},"hiddenSeries":false,"id":12,"interval":"","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket[5m])) by (le,type))","interval":"","legendFormat":"{{ type }}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"LogQL Latency","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":0,"y":17},"hiddenSeries":false,"id":13,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket[5m])) by (le,type))","interval":"","legendFormat":"{{ type }}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"LogQL Processed bytes","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"Bps","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":43,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Ruler","type":"row"},{"aliasColors":{},"breakPoint":"50%","combine":{"label":"Others","threshold":0},"datasource":{"type":"prometheus","uid":"mimir"},"fontSize":"80%","format":"short","gridPos":{"h":6,"w":7,"x":0,"y":24},"id":46,"interval":"","legend":{"show":true,"values":true},"legendType":"Right side","links":[],"maxDataPoints":3,"nullPointMode":"connected","pieType":"pie","pluginVersion":"7.3.1","strokeWidth":1,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"cortex_prometheus_rule_evaluations_total","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"Evaluations","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"cortex_prometheus_notifications_sent_total","instant":true,"interval":"1m","legendFormat":"Notifications","refId":"B"}],"title":"Rules Status ","type":"grafana-piechart-panel","valueName":"current"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":7,"y":24},"hiddenSeries":false,"id":28,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(rate(cortex_prometheus_notifications_latency_seconds[5m])) by (quantile)","interval":"","legendFormat":"TP {{quantile}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Notifications Latency","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":9,"x":15,"y":24},"hiddenSeries":false,"id":47,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(rate(cortex_prometheus_rule_evaluation_duration_seconds[5m])) by (quantile)","interval":"","legendFormat":"TP {{quantile}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Rule Evaluations","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":34,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Cache ","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":0,"y":31},"hiddenSeries":false,"id":45,"interval":"","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(cortex_cache_request_duration_seconds_bucket[5m])) by (le,name,method))","interval":"","legendFormat":"{{method}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Cache Request Duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":12,"y":31},"hiddenSeries":false,"id":29,"interval":"","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le,name,method))","interval":"","legendFormat":"{{name}} / {{method}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Cache Value Size bytes","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"decbytes","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":10,"x":0,"y":37},"hiddenSeries":false,"id":27,"legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"rate(cortex_cache_hits[5m])","interval":"","legendFormat":"{{name}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Hits Keys","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":7,"x":10,"y":37},"hiddenSeries":false,"id":30,"legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"cortex_cache_background_queue_length","interval":"","legendFormat":"{{name}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Background Queue Length","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"min":"0","show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":7,"x":17,"y":37},"hiddenSeries":false,"id":26,"legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"cortex_cache_fetched_keys","interval":"","legendFormat":"{{name}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Fetched Keys","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":43},"id":36,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Index","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":0,"y":44},"hiddenSeries":false,"id":38,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(cortex_cassandra_request_duration_seconds_bucket[5m])) by (le,operation))","interval":"","legendFormat":"{{operation}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Cassandra Request Durations","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":8,"y":44},"hiddenSeries":false,"id":39,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(cortex_chunk_store_chunks_per_query_bucket[5m])) by (le))","interval":"","legendFormat":"chunks","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunk Store Chunks","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"none","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":16,"y":44},"hiddenSeries":false,"id":54,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(cortex_table_manager_sync_duration_seconds_bucket[5m])) by (le))","interval":"","legendFormat":"","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Table-Manager Sync Durations","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","decimals":2,"format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":50},"id":19,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Distributor","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":6,"x":0,"y":51},"hiddenSeries":false,"id":20,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum by (tenant) (rate(loki_distributor_lines_received_total[5m]))","interval":"","legendFormat":"distributor / {{ tenant}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Received Lines / sec","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":6,"x":6,"y":51},"hiddenSeries":false,"id":15,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum(rate(loki_distributor_bytes_received_total[5m])) by (tenant)","interval":"","legendFormat":"distributor / {{tenant}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributor Received bytes / sec","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:837","format":"Bps","logBase":1,"show":true},{"$$hashKey":"object:838","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":6,"x":12,"y":51},"hiddenSeries":false,"id":17,"legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"rate(loki_distributor_ingester_appends_total[5m])","interval":"","legendFormat":"successed","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"loki_distributor_ingester_append_failures_total","interval":"","legendFormat":"failed","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"batch appends sent to ingesters / sec","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[],"unit":"short"},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":6,"x":18,"y":51},"hiddenSeries":false,"id":22,"interval":"","legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum by (status) (rate(loki_store_series_total[5m]))","instant":false,"interval":"","legendFormat":"{{status}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Store Series / sec","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:88","format":"short","logBase":1,"show":true},{"$$hashKey":"object:89","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":57},"id":66,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Ingster","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":0,"y":58},"hiddenSeries":false,"id":71,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket[5m])) by (le))","interval":"","legendFormat":"blocks","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Blocks / Chunk","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"none","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":8,"y":58},"hiddenSeries":false,"id":68,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket[5m])) by (le))","interval":"","legendFormat":"chunk size","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunk Size","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"decbytes","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":16,"y":58},"hiddenSeries":false,"id":72,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket[5m])) by (le))","interval":"","legendFormat":"ages","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunk Age","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":0,"y":64},"hiddenSeries":false,"id":70,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket[5m])) by (le))","interval":"","legendFormat":"ratio","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunk Compression Ratios","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"percent","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":8,"y":64},"hiddenSeries":false,"id":69,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket[5m])) by (le))","interval":"","legendFormat":"durations.","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunk Encode Time","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"s","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":6,"w":8,"x":16,"y":64},"hiddenSeries":false,"id":73,"interval":"","legend":{"alignAsTable":false,"avg":false,"current":true,"max":false,"min":false,"rightSide":false,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket[5m])) by (le))","interval":"","legendFormat":"lines","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Lines / Chunk","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:155","format":"none","logBase":1,"show":true},{"$$hashKey":"object:156","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":0,"y":70},"hiddenSeries":false,"id":60,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"rate(loki_ingester_chunks_created_total[5m])","interval":"","legendFormat":"create","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum by (reason) (rate(loki_ingester_chunks_flushed_total[5m]))","interval":"","legendFormat":"flush / {{ reason }}","refId":"B"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum by (fake) (rate(loki_ingester_chunks_stored_total[5m]))","interval":"","legendFormat":"stored","refId":"C"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"","interval":"","legendFormat":"","refId":"D"}],"thresholds":[],"timeRegions":[],"title":"Chunk Status","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":6,"x":12,"y":70},"hiddenSeries":false,"id":61,"legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"loki_ingester_memory_chunks","interval":"","legendFormat":"chunks","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunks in Memory","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":6,"x":18,"y":70},"hiddenSeries":false,"id":62,"legend":{"avg":false,"current":true,"max":false,"min":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"sum by (tenan) (loki_ingester_memory_streams)","interval":"","legendFormat":"streams","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Chunks in Streams","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"mimir"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":12,"x":0,"y":76},"hiddenSeries":false,"id":67,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.6","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"rate(loki_ingester_streams_created_total[5m])","interval":"","legendFormat":"create","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"rate(loki_ingester_streams_removed_total[5m])","interval":"","legendFormat":"delete","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Strams Stats","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:1092","format":"short","logBase":1,"show":true},{"$$hashKey":"object:1093","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"refresh":"5s","schemaVersion":37,"style":"dark","tags":["loki"],"templating":{"list":[]},"time":{"from":"now-15m","to":"now"},"timepicker":{"refresh_intervals":["1s","5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"","title":"Loki2.0 Global Metrics","uid":"MQHVDmtWk","version":1,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS JetStream.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS JetStream.json new file mode 100644 index 0000000..b8a2376 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS JetStream.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"NATS JetStream Dashboard","editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"id":5,"iteration":1642544826470,"links":[],"liveNow":false,"panels":[{"fieldConfig":{"defaults":{"decimals":3,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"#EAB839","value":0.75},{"color":"red","value":0.9}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":6,"w":4,"x":0,"y":0},"id":28,"options":{"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_jetstream_stats_storage{server_id=~\"$server\"})/sum(gnatsd_varz_jetstream_config_max_storage{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"},{"expr":"","interval":"","legendFormat":"","refId":"B"}],"title":"Storage Used","type":"gauge"},{"fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":5,"x":4,"y":0},"id":15,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_jetstream_stats_storage{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Total Storage Used","type":"stat"},{"fieldConfig":{"defaults":{"decimals":3,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"#EAB839","value":0.75},{"color":"red","value":0.9}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":6,"w":4,"x":9,"y":0},"id":31,"options":{"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_jetstream_stats_memory{server_id=~\"$server\"})/sum(gnatsd_varz_jetstream_config_max_memory{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"},{"expr":"","interval":"","legendFormat":"","refId":"B"}],"title":"Memory Used","type":"gauge"},{"fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":5,"x":13,"y":0},"id":32,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_jetstream_stats_memory{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Memory Used","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"min":0,"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":3,"w":6,"x":18,"y":0},"id":14,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_connections{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Connections","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":5,"x":4,"y":3},"id":30,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_jetstream_config_max_storage{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Max Storage","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":5,"x":13,"y":3},"id":33,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"sum(gnatsd_varz_jetstream_config_max_memory{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Total Memory","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":3,"w":6,"x":18,"y":3},"id":29,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"sum(jetstream_server_total_consumers{server_id=~\"$server\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Total Consumers","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":6},"id":19,"panels":[],"title":"Stream metrics","type":"row"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":6,"w":8,"x":0,"y":7},"id":17,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(jetstream_stream_total_bytes) by (stream_name)","interval":"","legendFormat":"{{stream_name}}","refId":"A"}],"title":"Stream data size","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":6,"w":8,"x":8,"y":7},"id":24,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(jetstream_stream_total_messages) by (stream_name)","interval":"","legendFormat":"{{stream_name}}","refId":"A"}],"title":"Stream message count","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"mps"},"overrides":[]},"gridPos":{"h":6,"w":8,"x":16,"y":7},"id":20,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(rate(jetstream_stream_total_messages{server_id=~\"$server\",stream_name=~\"$stream\"}[$__rate_interval])) by (stream_name)","hide":false,"interval":"","legendFormat":"{{stream_name}}","refId":"A"}],"title":"Message Rate (per second)","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":13},"id":23,"panels":[],"title":"Consumer Metrics","type":"row"},{"description":"Messages added \u0026 processed per minute per consumer","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"mps"},"overrides":[]},"gridPos":{"h":6,"w":24,"x":0,"y":14},"id":25,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(rate(jetstream_consumer_num_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}[$__rate_interval])+rate(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}[$__rate_interval])) by (consumer_name)","hide":false,"instant":false,"interval":"","intervalFactor":1,"legendFormat":"{{consumer_name}} +","refId":"B"},{"expr":"- sum(rate(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\",consumer_name=~\"$consumer\"}[$__rate_interval])) by (consumer_name)","hide":false,"interval":"","legendFormat":"{{consumer_name}} -","refId":"A"}],"title":"Messages per second (++/--)","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":6,"w":8,"x":0,"y":20},"id":21,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(jetstream_consumer_delivered_consumer_seq{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)","hide":false,"interval":"","legendFormat":"{{consumer_name}}","refId":"A"}],"title":"Total delivered messages","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":6,"w":8,"x":8,"y":20},"id":26,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(jetstream_consumer_num_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)","hide":false,"interval":"","legendFormat":"{{consumer_name}}","refId":"A"}],"title":"Pending messages","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":6,"w":8,"x":16,"y":20},"id":27,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"sum(jetstream_consumer_num_ack_pending{server_id=~\"$server\",stream_name=~\"$stream\",consumer_name=~\"$consumer\"}) by (consumer_name)","hide":false,"interval":"","legendFormat":"{{consumer_name}}","refId":"A"}],"title":"Message Acks Pending","type":"timeseries"}],"refresh":"10s","schemaVersion":34,"style":"dark","tags":[],"templating":{"list":[{"current":{},"datasource":"mimir","definition":"label_values(jetstream_server_total_streams, server_id)","hide":0,"includeAll":true,"label":"Server","multi":true,"name":"server","options":[],"query":{"query":"label_values(jetstream_server_total_streams, server_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{},"datasource":"mimir","definition":"label_values(jetstream_stream_last_seq, stream_name)","hide":0,"includeAll":true,"label":"Stream","multi":true,"name":"stream","options":[],"query":{"query":"label_values(jetstream_stream_last_seq, stream_name)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{},"datasource":"mimir","definition":"label_values(jetstream_consumer_num_pending, consumer_name)","hide":0,"includeAll":true,"label":"Consumer","multi":true,"name":"consumer","options":[],"query":{"query":"label_values(jetstream_consumer_num_pending, consumer_name)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"NATS JetStream","uid":"yQUo5l17k","version":2,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS Servers.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS Servers.json new file mode 100644 index 0000000..81c0db5 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/NATS Servers.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"NATS Server Dashboard","editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":6,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":10,"panels":[],"title":"OS Metrics","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":0,"y":1},"id":1,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_cpu","intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_cpu","refId":"A","step":4}],"title":"Server CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":1},"id":3,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_mem","intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_mem","refId":"A","step":4}],"title":"Server Memory","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":8},"id":11,"panels":[],"title":"Throughput","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":9},"id":7,"links":[],"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_in_bytes","intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_in_bytes","refId":"A","step":10}],"title":"Bytes In","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":9},"id":8,"links":[],"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_in_msgs","intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_in_msgs","refId":"A","step":10}],"title":"NATS Msgs In","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":9},"id":5,"links":[],"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_out_bytes","intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_out_bytes","refId":"A","step":10}],"title":"Bytes Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":9},"id":6,"links":[],"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_out_msgs","intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_out_msgs","refId":"A","step":10}],"title":"NATS Msgs Out","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":16},"id":12,"panels":[],"title":"Client Metrics","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"points","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":6,"scaleDistribution":{"type":"linear"},"showPoints":"always","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":17},"id":2,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_connections","intervalFactor":1,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_connections","refId":"A","step":2}],"title":"Connections","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"points","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":6,"scaleDistribution":{"type":"linear"},"showPoints":"always","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":17},"id":4,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_subscriptions","hide":false,"intervalFactor":2,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_subscriptions","refId":"A","step":4}],"title":"Subscriptions","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"points","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":2,"pointSize":6,"scaleDistribution":{"type":"linear"},"showPoints":"always","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":16,"y":17},"id":9,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"pluginVersion":"8.3.4","targets":[{"expr":"gnatsd_varz_slow_consumers","intervalFactor":1,"legendFormat":"{{server_id}}","metric":"gnatsd_varz_slow_consumers","refId":"A","step":2}],"title":"Slow Consumers","type":"timeseries"}],"refresh":false,"schemaVersion":34,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-5m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"NATS Servers","uid":"Usc6F_1nk","version":1,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik Realtime Stats.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik Realtime Stats.json new file mode 100644 index 0000000..3c03cde --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik Realtime Stats.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"description":"Visualize Traefik Health Metrics","editable":true,"gnetId":2240,"graphTooltip":0,"hideControls":false,"id":16,"links":[],"refresh":"30s","rows":[{"collapse":false,"height":288,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"id":1,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(traefik_entrypoint_requests_total) by (protocol)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ protocol }}","metric":"","refId":"A","step":240}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Total requests","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"none","label":"Count","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(traefik_backend_request_duration_seconds_sum) / sum(traefik_backend_requests_total) * 1000","format":"time_series","intervalFactor":2,"legendFormat":"Average response time (ms)","refId":"A","step":240}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average response time","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":5,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(traefik_backend_requests_total[5m]))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{requests}}","refId":"A","step":120}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests in last 5 minutes","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"General info","titleSize":"h6"},{"collapse":false,"height":412,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","decimals":0,"fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(traefik_backend_requests_total{protocol=~\"http|https\",code=\"200\"}[5m])) by (backend,method)","format":"time_series","intervalFactor":2,"legendFormat":"{{service}} {{method}}","refId":"A","step":240}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Successful Status Code Count (5min)","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","decimals":0,"fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(traefik_backend_requests_total{protocol=~\"http|https\",code!=\"200\"}[5m])) by (backend,method,code)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{backend}} {{method}} {{code}}","refId":"A","step":240}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Bad Status Code Count (5m)","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":true,"steppedLine":false,"targets":[{"expr":"process_open_fds{job=~\"load-balancer\"}","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ instance }}","refId":"A","step":240}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Used sockets","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["traefik","load-balancer"],"templating":{"list":[]},"time":{"from":"now-12h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"Traefik Realtime Stats","uid":"b73f37e0-666c-4c35-8844-9ecf3513cf9c","version":1} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik 2.2.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik 2.2.json new file mode 100644 index 0000000..5ff7aad --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/General/Traefik 2.2.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"$$hashKey":"object:112","builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","limit":100,"name":"Annotations \u0026 Alerts","showIn":0,"type":"dashboard"}]},"description":"Traefik dashboard (data from prometheus) (based on Traefik by Thomas Cheronneau https://grafana.com/grafana/dashboards/4475)","editable":true,"gnetId":12250,"graphTooltip":0,"id":14,"iteration":1588878844825,"links":[],"panels":[{"datasource":"mimir","gridPos":{"h":1,"w":24,"x":0,"y":0},"id":16,"title":"Global stats","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":1},"hiddenSeries":false,"id":14,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"dataLinks":[]},"percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(traefik_service_request_duration_seconds_sum{}) by (service) / sum(traefik_service_request_duration_seconds_count{}) by (service)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":" {{service}}","refId":"A","step":240}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Average response time by service","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:598","format":"s","label":null,"logBase":1,"max":null,"min":"0","show":true},{"$$hashKey":"object:599","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"breakPoint":"50%","cacheTimeout":null,"combine":{"label":"Others","threshold":0},"datasource":"mimir","fontSize":"80%","format":"short","gridPos":{"h":7,"w":6,"x":12,"y":1},"id":7,"interval":null,"legend":{"show":true,"values":true},"legendType":"Right side","links":[],"maxDataPoints":3,"nullPointMode":"connected","pieType":"pie","strokeWidth":1,"targets":[{"expr":"sum(rate(traefik_service_requests_total[5m])) by (service) ","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ service }}","refId":"A"}],"title":"Requests by service","type":"grafana-piechart-panel","valueName":"total"},{"aliasColors":{},"breakPoint":"50%","cacheTimeout":null,"combine":{"label":"Others","threshold":0},"datasource":"mimir","fontSize":"80%","format":"short","gridPos":{"h":7,"w":6,"x":18,"y":1},"id":8,"interval":null,"legend":{"show":true,"values":true},"legendType":"Right side","links":[],"maxDataPoints":3,"nullPointMode":"connected","pieType":"pie","strokeWidth":1,"targets":[{"expr":"sum(rate(traefik_entrypoint_requests_total{entrypoint =~ \"$entrypoint\"}[5m])) by (entrypoint) ","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ entrypoint }}","refId":"A"}],"title":"Requests by protocol","type":"grafana-piechart-panel","valueName":"total"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":8},"hiddenSeries":false,"id":5,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":true,"min":true,"rightSide":true,"show":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"nullPointMode":"null","options":{"dataLinks":[]},"percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=\"200\"}[5m])","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{method}} : {{code}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Status code 200 over 5min","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":8},"hiddenSeries":false,"id":6,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":true,"min":true,"rightSide":true,"show":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"nullPointMode":"null","options":{"dataLinks":[]},"percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code!=\"200\"}[5m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ method }} : {{code}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Others status code over 5min","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"collapsed":false,"datasource":"mimir","gridPos":{"h":1,"w":24,"x":0,"y":15},"id":10,"panels":[],"title":"$service stats","type":"row"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"mimir","format":"ms","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":7,"w":8,"x":0,"y":16},"id":4,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"$$hashKey":"object:451","name":"value to text","value":1},{"$$hashKey":"object:452","name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":true},"tableColumn":"","targets":[{"expr":"sum(traefik_service_request_duration_seconds_sum{service=\"$service\"}) / sum(traefik_service_requests_total{service=\"$service\"}) * 1000","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"$service response time","type":"singlestat","valueFontSize":"80%","valueMaps":[{"$$hashKey":"object:454","op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"breakPoint":"50%","cacheTimeout":null,"combine":{"label":"Others","threshold":0},"datasource":"mimir","fontSize":"80%","format":"short","gridPos":{"h":7,"w":8,"x":8,"y":16},"id":2,"interval":null,"legend":{"percentage":true,"show":true,"values":true},"legendType":"Right side","links":[],"maxDataPoints":3,"nullPointMode":"connected","pieType":"pie","strokeWidth":1,"targets":[{"expr":"traefik_service_requests_total{service=\"$service\"}","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{method}} : {{code}}","refId":"A"}],"title":"$service return code","type":"grafana-piechart-panel","valueName":"current"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"mimir","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":16,"y":16},"hiddenSeries":false,"id":3,"legend":{"alignAsTable":true,"avg":true,"current":false,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"nullPointMode":"null","options":{"dataLinks":[]},"percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(traefik_service_requests_total{service=\"$service\"}[5m]))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"Total requests $service","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Total requests over 5min $service","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:509","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:510","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"refresh":"5m","schemaVersion":22,"style":"dark","tags":["prometheus","traefik"],"templating":{"list":[{"allValue":null,"current":{},"datasource":"mimir","definition":"label_values(service)","hide":0,"includeAll":false,"index":-1,"label":null,"multi":false,"name":"service","options":[],"query":"label_values(service)","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"mimir","definition":"","hide":0,"includeAll":true,"index":-1,"label":null,"multi":true,"name":"entrypoint","options":[],"query":"label_values(entrypoint)","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Traefik 2.2","uid":"O23g2BeWk","variables":{"list":[]},"version":1} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Ingress/Traefik.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Ingress/Traefik.json new file mode 100644 index 0000000..7d61055 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Ingress/Traefik.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Traefik dashboard prometheus","editable":true,"fiscalYearStartMonth":0,"gnetId":4475,"graphTooltip":0,"id":49,"links":[],"liveNow":false,"panels":[{"collapsed":true,"datasource":{"type":"prometheus","uid":"$datasource"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":10,"panels":[{"datasource":{"type":"datasource","uid":"-- Dashboard --"},"fieldConfig":{"defaults":{"color":{"fixedColor":"purple","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":2,"w":8,"x":0,"y":1},"id":17,"interval":"$agg_interval","links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"datasource","uid":"-- Dashboard --"},"panelId":15,"refId":"A"}],"title":"Load","transformations":[{"id":"filterFieldsByName","options":{"include":{"pattern":"Total"}}},{"id":"reduce","options":{"includeTimeField":false,"mode":"seriesToRows","reducers":["mean","sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Mean":"Average load","Total":"Total"}}}],"type":"stat"},{"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":2,"w":8,"x":8,"y":1},"id":4,"links":[],"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum(traefik_service_request_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"})\n/\nsum(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"})","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"Average response time","range":true,"refId":"A"}],"title":"Average response time","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"red","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percent"},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"unit","value":"short"},{"id":"decimals","value":0}]}]},"gridPos":{"h":2,"w":8,"x":16,"y":1},"id":16,"interval":"$agg_interval","links":[],"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\", code=~\"(4|5).+\"}[$__interval] offset -$__interval))","format":"time_series","instant":false,"legendFormat":"Errors","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval))","format":"time_series","hide":false,"instant":false,"legendFormat":"Total","range":true,"refId":"B"}],"title":"Errors","transformations":[{"id":"calculateField","options":{"alias":"Rate","binary":{"left":"Errors","operator":"/","reducer":"sum","right":"Total"},"mode":"binary","reduce":{"include":[],"reducer":"sum"}}},{"id":"organize","options":{"excludeByName":{"Total":true},"indexByName":{},"renameByName":{"Errors":"Total"}}}],"type":"stat"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":-1,"drawStyle":"line","fillOpacity":7,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.drawStyle","value":"line"},{"id":"custom.fillOpacity","value":0},{"id":"color","value":{"fixedColor":"#b877d9","mode":"fixed"}},{"id":"custom.stacking","value":{"group":"A","mode":"none"}}]}]},"gridPos":{"h":7,"w":8,"x":0,"y":3},"id":15,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["mean","max","min","sum"],"displayMode":"list","placement":"bottom","showLegend":true,"sortBy":"Total","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (service) (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval))\u003e 0","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{service}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$datasource"},"editorMode":"code","expr":"sum by () (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval)) \u003e 0","hide":false,"interval":"","legendFormat":"Total","range":true,"refId":"B"}],"title":"Load / $__interval","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"","fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":5,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":3},"id":14,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["mean","max","min","lastNotNull"],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum by(service) (traefik_service_request_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"})\n/\nsum by(service)(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"})","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Average response time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"Ratio of 4xx and 5xx HTTP responses to all calls.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":-1,"drawStyle":"line","fillOpacity":14,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":16,"y":3},"id":13,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["mean","max","min","lastNotNull"],"displayMode":"list","placement":"bottom","showLegend":true,"sortBy":"Total","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (service) (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\", code=~\"(4|5).+\"}[$__interval] offset -$__interval))/\nsum by (service) (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval)\u003e0)*100","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{ service }}","range":true,"refId":"A"}],"title":"Errors rate","transformations":[],"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"Load distribution by method, code over $__interval","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":1,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.drawStyle","value":"line"},{"id":"custom.fillOpacity","value":0},{"id":"color","value":{"fixedColor":"#b877d9","mode":"fixed"}},{"id":"custom.stacking","value":{"group":"A","mode":"none"}}]},{"matcher":{"id":"byName","options":"HTTP 100-199"},"properties":[{"id":"color","value":{"fixedColor":"purple","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"HTTP 200-299"},"properties":[{"id":"color","value":{"fixedColor":"green","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"HTTP 300-399"},"properties":[{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"HTTP 400-499"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"HTTP 500-599"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"id":3,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["mean","max","min","sum"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Total","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (alias) (\nlabel_replace(\n(increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\", code!=\"\"}[$__interval] offset -$__interval )) \u003e 0,\n \"alias\", \"HTTP ${1}00-${1}99\", \"code\", \"(.).+\"\n))","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{alias}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$datasource"},"editorMode":"code","expr":"sum by () (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval)) \u003e 0","hide":false,"interval":"","legendFormat":"Total","range":true,"refId":"B"}],"title":"Load by HTTP codes/$__interval","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"Load distribution by method, code over $__interval","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":1,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.drawStyle","value":"line"},{"id":"custom.stacking","value":{"group":"A","mode":"none"}},{"id":"color","value":{"fixedColor":"purple","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"id":19,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["mean","max","min","sum"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Total","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (method) (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\", code!=\"\"}[$__interval] offset -$__interval)) \u003e 0","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{alias}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$datasource"},"editorMode":"code","expr":"sum by () (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval)) \u003e 0","hide":false,"interval":"","legendFormat":"Total","range":true,"refId":"B"}],"title":"Load by method/$__interval","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"refId":"A"}],"title":"Service stats ($service)","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"$datasource"},"gridPos":{"h":1,"w":24,"x":0,"y":1},"id":12,"panels":[{"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":1,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":0,"y":2},"id":5,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["lastNotNull","max","min","mean","sum"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Total","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (method, code) (increase(traefik_entrypoint_requests_total{job=~\"$job\", instance=~\"$instance\", entrypoint=~\"$entrypoint\",code=~\"2.+\"}[$__interval] offset -$__interval)) \u003e 0","format":"time_series","intervalFactor":1,"legendFormat":"{{method}}: {{code}}","range":true,"refId":"A"}],"title":"Status codes 2xx/$__interval","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":1,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":2},"id":6,"interval":"$agg_interval","links":[],"options":{"legend":{"calcs":["lastNotNull","max","min","mean","sum"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Total","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.4","targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (method, code) (increase(traefik_entrypoint_requests_total{job=~\"$job\", instance=~\"$instance\", entrypoint=~\"$entrypoint\",code!~\"2.+\"}[$__interval] offset -$__interval)) \u003e 0","format":"time_series","intervalFactor":1,"legendFormat":"{{ method }} : {{code}}","range":true,"refId":"A"}],"title":"Others status codes/$__interval","type":"timeseries"},{"aliasColors":{},"breakPoint":"50%","combine":{"label":"Others","threshold":0},"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"unit":"short"},"overrides":[]},"fontSize":"80%","format":"short","gridPos":{"h":7,"w":12,"x":0,"y":9},"id":7,"interval":"$agg_interval","links":[],"nullPointMode":"connected","options":{"legend":{"displayMode":"table","placement":"right","showLegend":true,"values":["value","percent"]},"pieType":"pie","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"tooltip":{"mode":"multi","sort":"desc"}},"pieType":"pie","strokeWidth":1,"targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (service) (increase(traefik_service_requests_total{job=~\"$job\", instance=~\"$instance\", service=~\"$service\"}[$__interval] offset -$__interval)) \u003e 0","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ service }}","range":true,"refId":"A"}],"title":"Requests by service","type":"piechart","valueName":"total"},{"aliasColors":{},"breakPoint":"50%","combine":{"label":"Others","threshold":0},"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"unit":"short"},"overrides":[]},"fontSize":"80%","format":"short","gridPos":{"h":7,"w":12,"x":12,"y":9},"id":8,"interval":"$agg_interval","links":[],"nullPointMode":"connected","options":{"legend":{"displayMode":"table","placement":"right","showLegend":true,"values":["value","percent"]},"pieType":"pie","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"tooltip":{"mode":"multi","sort":"desc"}},"pieType":"pie","strokeWidth":1,"targets":[{"datasource":{"uid":"$datasource"},"editorMode":"code","expr":"sum by (protocol, entrypoint) (increase(traefik_entrypoint_requests_total{job=~\"$job\", instance=~\"$instance\", entrypoint=~\"$entrypoint\"}[$__interval] offset -$__interval)) \u003e 0","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{entrypoint}}: {{ protocol }}","range":true,"refId":"A"}],"title":"Requests by entrypoints and protocol","type":"piechart","valueName":"total"}],"targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"refId":"A"}],"title":"Global stats","type":"row"}],"refresh":"30s","schemaVersion":38,"style":"dark","tags":["traefik-integration"],"templating":{"list":[{"hide":0,"includeAll":false,"label":"Data Source","multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":".+","current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(traefik_service_requests_total,job)","hide":0,"includeAll":true,"label":"job","multi":true,"name":"job","options":[],"query":{"query":"label_values(traefik_service_requests_total,job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"allValue":".+","current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(traefik_service_requests_total{job=~\"$job\"},instance)","hide":0,"includeAll":true,"label":"instance","multi":true,"name":"instance","options":[],"query":{"query":"label_values(traefik_service_requests_total{job=~\"$job\"},instance)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{"selected":true,"text":["All"],"value":["$__all"]},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(traefik_service_requests_total{job=~\"$job\",instance=~\"$instance\"},service)","hide":0,"includeAll":true,"label":"Service","multi":true,"name":"service","options":[],"query":{"query":"label_values(traefik_service_requests_total{job=~\"$job\",instance=~\"$instance\"},service)","refId":"StandardVariableQuery"},"refresh":1,"regex":"/.*@.*/","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":["All"],"value":["$__all"]},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(traefik_entrypoint_requests_total{job=~\"$job\",instance=~\"$instance\"},entrypoint)","hide":0,"includeAll":true,"label":"Entrypoint","multi":true,"name":"entrypoint","options":[],"query":{"query":"label_values(traefik_entrypoint_requests_total{job=~\"$job\",instance=~\"$instance\"},entrypoint)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"auto":true,"auto_count":100,"auto_min":"1m","current":{"selected":false,"text":"auto","value":"$__auto_interval_agg_interval"},"description":"Desired group interval","hide":0,"label":"Aggregation interval","name":"agg_interval","options":[{"selected":true,"text":"auto","value":"$__auto_interval_agg_interval"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"5m","value":"5m"},{"selected":false,"text":"10m","value":"10m"},{"selected":false,"text":"1h","value":"1h"},{"selected":false,"text":"1d","value":"1d"}],"query":"1m,5m,10m,1h,1d","queryValue":"","refresh":2,"skipUrlSync":false,"type":"interval"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Traefik","uid":"qPdAviJmz","version":4,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Full.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Full.json new file mode 100644 index 0000000..1705c1c --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Full.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"$$hashKey":"object:1058","builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"gnetId":1860,"graphTooltip":0,"id":7,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"GitHub","type":"link","url":"https://github.com/rfmoz/grafana-dashboards"},{"icon":"external link","tags":[],"targetBlank":true,"title":"Grafana","type":"link","url":"https://grafana.com/grafana/dashboards/1860"}],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":261,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Quick CPU / Mem / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Busy state of all CPU cores together (15 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":0,"y":1},"id":19,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"avg(node_load1{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100","hide":false,"intervalFactor":1,"range":true,"refId":"A","step":240}],"title":"Sys Load (1m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Busy state of all CPU cores together (5 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":3,"y":1},"id":155,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"avg(node_load5{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100","format":"time_series","hide":false,"intervalFactor":1,"refId":"A","step":240}],"title":"Sys Load (5m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Busy state of all CPU cores together (5 min average)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":6,"y":1},"id":320,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"avg(node_load15{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100","format":"time_series","hide":false,"intervalFactor":1,"range":true,"refId":"A","step":240}],"title":"Sys Load (15m avg)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Non available RAM memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":9,"y":1},"hideTimeOverride":false,"id":16,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100","format":"time_series","hide":true,"intervalFactor":1,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"refId":"B","step":240}],"title":"RAM Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Used Swap","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":10},{"color":"rgba(245, 54, 54, 0.9)","value":25}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":12,"y":1},"id":21,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100","intervalFactor":1,"refId":"A","step":240}],"title":"SWAP Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Used Root FS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":15,"y":1},"id":154,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})","format":"time_series","intervalFactor":1,"refId":"A","step":240}],"title":"Root FS Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Total number of CPU cores","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":1},"id":14,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))","interval":"","intervalFactor":1,"legendFormat":"","refId":"A","step":240}],"title":"CPU Cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"System uptime","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":2,"w":4,"x":20,"y":1},"hideTimeOverride":true,"id":15,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}","intervalFactor":1,"refId":"A","step":240}],"title":"Uptime","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Total RootFS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":70},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":3},"id":23,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}","format":"time_series","hide":false,"intervalFactor":1,"refId":"A","step":240}],"title":"RootFS Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Total RAM","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":20,"y":3},"id":75,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","intervalFactor":1,"refId":"A","step":240}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Total SWAP","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":22,"y":3},"id":18,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}","intervalFactor":1,"refId":"A","step":240}],"title":"SWAP Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Busy state of all CPU cores together","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":0,"y":5},"id":20,"links":[],"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.4.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100","hide":false,"intervalFactor":1,"legendFormat":"","range":true,"refId":"A","step":240}],"title":"CPU Busy","type":"gauge"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":9},"id":263,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Basic CPU / Mem / Net / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Basic CPU info","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy System"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy User"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Other"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":10},"id":77,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy System","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Busy User","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Iowait","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy IRQs","range":true,"refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Busy Other","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Idle","range":true,"refId":"F","step":240}],"title":"CPU Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Basic memory usage","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"SWAP Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap Used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Total"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]},{"matcher":{"id":"byName","options":"RAM Cache + Buffer"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM Free"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Avaliable"},"properties":[{"id":"color","value":{"fixedColor":"#DEDAF7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":7,"w":12,"x":12,"y":10},"id":78,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Total","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"RAM Used","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Cache + Buffer","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"RAM Free","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","intervalFactor":1,"legendFormat":"SWAP Used","refId":"E","step":240}],"title":"Memory Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Basic network info per interface","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"Recv_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_eth2"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#CCA300","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"recv_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_bytes_lo"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_drop_lo"},"properties":[{"id":"color","value":{"fixedColor":"#967302","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"trans_errs_lo"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":17},"id":74,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"recv {{device}}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"trans {{device}} ","refId":"B","step":240}],"title":"Network Traffic Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Disk space used of all filesystems mounted","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":17},"id":152,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used Basic","type":"timeseries"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":265,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Idle - Waiting for something to happen"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Iowait - Waiting for I/O to complete"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Irq - Servicing interrupts"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Nice - Niced processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Softirq - Servicing softirqs"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Steal - Time spent in other operating systems when running in a virtualized environment"},"properties":[{"id":"color","value":{"fixedColor":"#FCE2DE","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"System - Processes executing in kernel mode"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"User - Normal processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#5195CE","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":7},"id":3,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":250},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"System - Processes executing in kernel mode","range":true,"refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"User - Normal processes executing in user mode","range":true,"refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Nice - Niced processes executing in user mode","range":true,"refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Iowait - Waiting for I/O to complete","range":true,"refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Irq - Servicing interrupts","range":true,"refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Softirq - Servicing softirqs","range":true,"refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","intervalFactor":1,"legendFormat":"Steal - Time spent in other operating systems when running in a virtualized environment","range":true,"refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Idle - Waiting for something to happen","range":true,"refId":"J","step":240}],"title":"CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap - Swap memory usage"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused - Free memory unassigned"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Hardware Corrupted - *./"},"properties":[{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":7},"id":24,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Apps - Memory used by user-space applications","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"PageTables - Memory used to map between virtual and physical memory addresses","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Cache - Parked file data (file content) cache","refId":"E","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Buffers - Block device (e.g. harddisk) cache","refId":"F","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Unused - Free memory unassigned","refId":"G","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Swap - Swap space used","refId":"H","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working","refId":"I","step":240}],"title":"Memory Stack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bits out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":19},"id":84,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":12,"y":19},"id":156,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A","step":240}],"title":"Disk Space Used","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":31},"id":229,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*read*./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":31},"id":42,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Successfully written bytes","refId":"B","step":240}],"title":"I/O Usage Read / Write","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"io time"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"hidden"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":43},"id":127,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"I/O Utilization","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"percentage","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":3,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":1,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/^Guest - /"},"properties":[{"id":"color","value":{"fixedColor":"#5195ce","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/^GuestNice - /"},"properties":[{"id":"color","value":{"fixedColor":"#c15c17","mode":"fixed"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":43},"id":319,"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"Guest - Time spent running a virtual CPU for a guest operating system","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))","hide":false,"legendFormat":"GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)","range":true,"refId":"B"}],"title":"CPU spent seconds in guests (VMs)","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"CPU / Memory / Net / Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":25},"id":266,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":38},"id":136,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary","refId":"B","step":240}],"title":"Memory Active / Inactive","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*CommitLimit - *./"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":38},"id":135,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Committed_AS - Amount of memory presently allocated on the system","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"CommitLimit - Amount of memory currently available to be allocated on the system","refId":"B","step":240}],"title":"Memory Commited","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":48},"id":191,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_file - File-backed memory on inactive LRU list","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_file - File-backed memory on active LRU list","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs","refId":"D","step":240}],"title":"Memory Active / Inactive Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":48},"id":130,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Writeback - Memory which is actively being written back to disk","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"WritebackTmp - Memory used by FUSE for temporary writeback buffers","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Dirty - Memory which is waiting to get written back to the disk","refId":"C","step":240}],"title":"Memory Writeback and Dirty","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byName","options":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":58},"id":138,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Mapped - Used memory in mapped pages files which have been mmaped, such as libraries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Shmem - Used shared memory (shared between several processes, thus including RAM disks)","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ShmemPmdMapped - Ammount of shared (shmem/tmpfs) memory backed by huge pages","refId":"D","step":240}],"title":"Memory Shared and Mapped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":58},"id":131,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"SReclaimable - Part of Slab, that might be reclaimed, such as caches","refId":"B","step":240}],"title":"Memory Slab","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":68},"id":70,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocChunk - Largest contigious block of vmalloc area which is free","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocTotal - Total size of vmalloc memory area","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"VmallocUsed - Amount of vmalloc area which is used","refId":"C","step":240}],"title":"Memory Vmalloc","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":68},"id":159,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Bounce - Memory used for block device bounce buffers","refId":"A","step":240}],"title":"Memory Bounce","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Inactive *./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":78},"id":129,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonHugePages - Memory in anonymous huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"AnonPages - Memory in user pages not backed by files","refId":"B","step":240}],"title":"Memory Anonymous","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":78},"id":160,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"KernelStack - Kernel memory stack. This is not reclaimable","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PerCPU - Per CPU memory allocated dynamically by loadable modules","refId":"B","step":240}],"title":"Memory Kernel / CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":88},"id":140,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Free - Huge pages in the pool that are not yet allocated","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages","refId":"C","step":240}],"title":"Memory HugePages Counter","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":88},"id":71,"links":[],"options":{"legend":{"calcs":["lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"HugePages - Total size of the pool of huge pages","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Hugepagesize - Huge Page size","refId":"B","step":240}],"title":"Memory HugePages Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":98},"id":128,"links":[],"options":{"legend":{"calcs":["mean","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"DirectMap1G - Amount of pages mapped as this size","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap2M - Amount of pages mapped as this size","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"DirectMap4K - Amount of pages mapped as this size","refId":"C","step":240}],"title":"Memory DirectMap","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":98},"id":137,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"MLocked - Size of pages locked to memory using the mlock() system call","refId":"B","step":240}],"title":"Memory Unevictable and MLocked","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":108},"id":132,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage","refId":"A","step":240}],"title":"Memory NFS","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Memory Meminfo","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":267,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":25},"id":176,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesin - Page in operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pagesout - Page out operations","refId":"B","step":240}],"title":"Memory Pages In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"pages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":25},"id":22,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpin - Pages swapped in","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pswpout - Pages swapped out","refId":"B","step":240}],"title":"Memory Pages Swap In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"faults","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Pgfault - Page major and minor fault operations"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":35},"id":175,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgfault - Page major and minor fault operations","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgmajfault - Major page fault operations","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Pgminfault - Minor page fault operations","refId":"C","step":240}],"title":"Memory Page Faults","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Swap"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":35},"id":307,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"oom killer invocations ","refId":"A","step":240}],"title":"OOM Killer","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Memory Vmstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":293,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":40},"id":260,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Estimated error in seconds","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Time offset in between local system and reference clock","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum error in seconds","refId":"C","step":240}],"title":"Time Syncronized Drift","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":40},"id":291,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Phase-locked loop time adjust","refId":"A","step":240}],"title":"Time PLL Adjust","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Variation*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":168,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_sync_status{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Is clock synchronized to a reliable server (1 = yes, 0 = no)","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Local clock frequency adjustment","refId":"B","step":240}],"title":"Time Syncronized Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":294,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Seconds between clock ticks","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"International Atomic Time (TAI) offset","refId":"B","step":240}],"title":"Time Misc","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"System Timesync","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":312,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":27},"id":62,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_procs_blocked{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes blocked waiting for I/O to complete","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_procs_running{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Processes in runnable state","refId":"B","step":240}],"title":"Processes Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":27},"id":315,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_processes_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ state }}","refId":"A","step":240}],"title":"Processes State","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"forks / sec","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":37},"id":148,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Processes forks second","refId":"A","step":240}],"title":"Processes Forks","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max.*/"},"properties":[{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":37},"id":149,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Processes virtual memory size in bytes","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Maximum amount of virtual memory available in bytes","refId":"D","step":240}],"title":"Processes Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"PIDs limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":47},"id":313,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_processes_pids{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Number of PIDs","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_processes_max_processes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PIDs limit","refId":"B","step":240}],"title":"PIDs Number and Limit","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*waiting.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":47},"id":305,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent running a process","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }} - seconds spent by processing waiting for this CPU","refId":"B","step":240}],"title":"Process schedule stats Running / Waiting","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Threads limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":57},"id":314,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_processes_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Allocated threads","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_processes_max_threads{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Threads limit","refId":"B","step":240}],"title":"Threads Number and Limit","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"System Processes","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":269,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":8,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"Context switches","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Interrupts","refId":"B","step":240}],"title":"Context Switches / Interrupts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":7,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_load1{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 1m","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_load5{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 5m","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_load15{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Load 15m","refId":"C","step":240}],"title":"System Load","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":259,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ type }} - {{ info }}","refId":"A","step":240}],"title":"Interrupts Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":52},"id":306,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{ cpu }}","refId":"A","step":240}],"title":"Schedule timeslices executed by each cpu","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":62},"id":151,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_entropy_available_bits{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Entropy available to random number generators","refId":"A","step":240}],"title":"Entropy","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":62},"id":308,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Time spent","refId":"A","step":240}],"title":"CPU time spent in user and system contexts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":72},"id":64,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"process_max_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Maximum open file descriptors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"process_open_fds{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"Open file descriptors","refId":"B","step":240}],"title":"File Descriptors","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"System Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":304,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"temperature","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"celsius"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":158,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} temp","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Alarm","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Critical Historical","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"{{ chip }} {{ sensor }} Max","refId":"E","step":240}],"title":"Hardware temperature monitor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":300,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Current {{ name }} in {{ type }}","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Max {{ name }} in {{ type }}","refId":"B","step":240}],"title":"Throttle cooling device","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":302,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_power_supply_online{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{ power_supply }} online","refId":"A","step":240}],"title":"Power supply","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Hardware Misc","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":296,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":297,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ name }} Connections","refId":"A","step":240}],"title":"Systemd Sockets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Failed"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#FF9830","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#73BF69","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Deactivating"},"properties":[{"id":"color","value":{"fixedColor":"#FFCB7D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Activating"},"properties":[{"id":"color","value":{"fixedColor":"#C8F2C2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":298,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Activating","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Active","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Deactivating","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Failed","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Inactive","refId":"E","step":240}],"title":"Systemd Units State","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Systemd","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":32},"id":270,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"The number (after merges) of I/O requests completed per second for the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IO read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":9,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":4,"legendFormat":"{{device}} - Reads completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Writes completed","refId":"B","step":240}],"title":"Disk IOps Completed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"The number of bytes read from or written to the device per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":33,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":4,"legendFormat":"{{device}} - Read bytes","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Written bytes","refId":"B","step":240}],"title":"Disk R/W Data","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"time. read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":41},"id":37,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":4,"legendFormat":"{{device}} - Read wait time avg","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{device}} - Write wait time avg","refId":"B","step":240}],"title":"Disk Average Wait Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"The average queue length of the requests that were issued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"aqu-sz","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":41},"id":35,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}}","refId":"A","step":240}],"title":"Average Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"The number of read and write requests merged per second that were queued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"I/Os","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":51},"id":133,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Read merged","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","intervalFactor":1,"legendFormat":"{{device}} - Write merged","refId":"B","step":240}],"title":"Disk R/W Merged","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"%util","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":51},"id":36,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - discard","refId":"B","step":240}],"title":"Time Spent Doing I/Os","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Outstanding req.","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":61},"id":34,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_disk_io_now{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":4,"legendFormat":"{{device}} - IO now","refId":"A","step":240}],"title":"Instantaneous Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"IOs","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*sda_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda2_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BA43A9","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sda3_.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F4D598","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdb3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#962D82","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdc3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#9AC48A","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#65C5DB","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9934E","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde1.*/"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sdd2.*/"},"properties":[{"id":"color","value":{"fixedColor":"#FCEACA","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*sde3.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F9E2D2","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":61},"id":301,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":4,"legendFormat":"{{device}} - Discards completed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Discards merged","refId":"B","step":240}],"title":"Disk IOps Discards completed / merged","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Storage Disk","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":33},"id":271,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":46},"id":43,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Available","metric":"","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Size","refId":"C","step":240}],"title":"Filesystem space available","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":46},"id":41,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - Free file nodes","refId":"A","step":240}],"title":"File Nodes Free","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"files","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":56},"id":28,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filefd_maximum{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":4,"legendFormat":"Max open files","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filefd_allocated{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"Open files","refId":"B","step":240}],"title":"File Descriptor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"file Nodes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":56},"id":219,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{mountpoint}} - File nodes total","refId":"A","step":240}],"title":"File Nodes Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"/ ReadOnly"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":66},"id":44,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}","format":"time_series","intervalFactor":1,"legendFormat":"{{mountpoint}} - ReadOnly","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{mountpoint}} - Device error","refId":"B","step":240}],"title":"Filesystem in ReadOnly / Error","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Storage Filesystem","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":34},"id":272,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byName","options":"receive_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"receive_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_eth0"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"transmit_packets_lo"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":33},"id":60,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Receive","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{device}} - Transmit","refId":"B","step":240}],"title":"Network Traffic by Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":33},"id":142,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive errors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Rransmit errors","refId":"B","step":240}],"title":"Network Traffic Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":143,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive drop","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit drop","refId":"B","step":240}],"title":"Network Traffic Drop","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":141,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive compressed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit compressed","refId":"B","step":240}],"title":"Network Traffic Compressed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":146,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive multicast","refId":"A","step":240}],"title":"Network Traffic Multicast","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":53},"id":144,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Receive fifo","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit fifo","refId":"B","step":240}],"title":"Network Traffic Fifo","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":63},"id":145,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{device}} - Receive frame","refId":"A","step":240}],"title":"Network Traffic Frame","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":63},"id":231,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Statistic transmit_carrier","refId":"A","step":240}],"title":"Network Traffic Carrier","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Trans.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":73},"id":232,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} - Transmit colls","refId":"A","step":240}],"title":"Network Traffic Colls","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"NF conntrack limit"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":73},"id":61,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack entries","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"NF conntrack limit","refId":"B","step":240}],"title":"NF Contrack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Entries","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":83},"id":230,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_arp_entries{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - ARP entries","refId":"A","step":240}],"title":"ARP Entries","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":83},"id":288,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Bytes","refId":"A","step":240}],"title":"MTU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":93},"id":280,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_network_speed_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Speed","refId":"A","step":240}],"title":"Speed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":93},"id":289,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{ device }} - Interface transmit queue length","refId":"A","step":240}],"title":"Queue Length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"packetes drop (-) / process (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Dropped.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":103},"id":290,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Processed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Dropped","refId":"B","step":240}],"title":"Softnet Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":103},"id":310,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"CPU {{cpu}} - Squeezed","refId":"A","step":240}],"title":"Softnet Out of Quota","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":113},"id":309,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{interface}} - Operational state UP","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_network_carrier{instance=\"$node\",job=\"$job\"}","format":"time_series","instant":false,"legendFormat":"{{device}} - Physical link state","refId":"B"}],"title":"Network Operational Status","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Network Traffic","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":35},"id":273,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":48},"id":63,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_alloc - Allocated sockets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_inuse - Tcp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":true,"interval":"","intervalFactor":1,"legendFormat":"TCP_mem - Used memory for tcp","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_orphan - Orphan sockets","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCP_tw - Sockets wating close","refId":"E","step":240}],"title":"Sockstat TCP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":48},"id":124,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDPLITE_inuse - Udplite sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_inuse - Udp sockets currently in use","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"UDP_mem - Used memory for udp","refId":"C","step":240}],"title":"Sockstat UDP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":58},"id":125,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"FRAG_inuse - Frag sockets currently in use","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RAW_inuse - Raw sockets currently in use","refId":"C","step":240}],"title":"Sockstat FRAG / RAW","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":58},"id":220,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - TCP sockets in that state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"mem_bytes - UDP sockets in that state","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}","interval":"","intervalFactor":1,"legendFormat":"FRAG_memory - Used memory for frag","refId":"C"}],"title":"Sockstat Memory Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"sockets","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":68},"id":126,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Sockets_used - Sockets currently in use","refId":"A","step":240}],"title":"Sockstat Used","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Network Sockstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":36},"id":274,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"octects out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":49},"id":221,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InOctets - Received octets","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"OutOctets - Sent octets","refId":"B","step":240}],"title":"Netstat IP In / Out Octets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":49},"id":81,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Forwarding - IP forwarding","refId":"A","step":240}],"title":"Netstat IP Forwarding","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":59},"id":115,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors","refId":"B","step":240}],"title":"ICMP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"messages out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":59},"id":50,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)","refId":"A","step":240}],"title":"ICMP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":69},"id":55,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InDatagrams - Datagrams received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutDatagrams - Datagrams sent","refId":"B","step":240}],"title":"UDP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":69},"id":109,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"InErrors - UDP Datagrams that could not be delivered to an application","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"NoPorts - UDP Datagrams received on a port with no listener","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrors Lite - UDPLite Datagrams that could not be delivered to an application","refId":"C"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"RcvbufErrors - UDP buffer errors received","refId":"D","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"SndbufErrors - UDP buffer errors send","refId":"E","step":240}],"title":"UDP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"datagrams out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":79},"id":299,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"InSegs - Segments received, including those received in error. This count includes segments received on currently established connections","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets","refId":"B","step":240}],"title":"TCP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":79},"id":104,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenOverflows - Times the listen queue of a socket overflowed","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"ListenDrops - SYNs to LISTEN sockets ignored","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits","refId":"C","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets","refId":"D"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"InErrs - Segments received in error (e.g., bad TCP checksums)","refId":"E"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])","interval":"","legendFormat":"OutRsts - Segments sent with RST flag","refId":"F"}],"title":"TCP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*MaxConn *./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":89},"id":85,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")","refId":"B","step":240}],"title":"TCP Connections","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Sent.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":89},"id":91,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesFailed - Invalid SYN cookies received","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesRecv - SYN cookies received","refId":"B","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"SyncookiesSent - SYN cookies sent","refId":"C","step":240}],"title":"TCP SynCookie","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":99},"id":82,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state","refId":"B","step":240}],"title":"TCP Direct Transition","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Network Netstat","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":1,"w":24,"x":0,"y":37},"id":279,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"seconds","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":50},"id":40,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape duration","refId":"A","step":240}],"title":"Node Exporter Scrape Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"counter","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineStyle":{"fill":"solid"},"lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*error.*/"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":50},"id":157,"links":[],"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_scrape_collector_success{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape success","refId":"A","step":240},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{collector}} - Scrape textfile error (1 = true)","refId":"B","step":240}],"title":"Node Exporter Scrape","type":"timeseries"}],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Node Exporter","type":"row"}],"refresh":"5s","revision":1,"schemaVersion":38,"style":"dark","tags":["linux"],"templating":{"list":[{"current":{"selected":false,"text":"default","value":"default"},"hide":0,"includeAll":false,"label":"datasource","multi":false,"name":"DS_PROMETHEUS","options":[],"query":"prometheus","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"mimir"},"definition":"","hide":0,"includeAll":false,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(node_uname_info, job)","refId":"Prometheus-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{},"datasource":{"type":"prometheus","uid":"mimir"},"definition":"label_values(node_uname_info{job=\"$job\"}, instance)","hide":0,"includeAll":false,"label":"Host:","multi":false,"name":"node","options":[],"query":{"query":"label_values(node_uname_info{job=\"$job\"}, instance)","refId":"Prometheus-node-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"},"hide":2,"includeAll":false,"multi":false,"name":"diskdevices","options":[{"selected":true,"text":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","value":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+"}],"query":"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-24h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"Node Exporter Full","uid":"rYdddlPWk","version":4,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -Nodes.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -Nodes.json new file mode 100644 index 0000000..16e80f6 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -Nodes.json @@ -0,0 +1 @@ +{"annotations":{"list":[]},"editable":false,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":45,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"integrations/node_exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"integrations/node_exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n","format":"time_series","intervalFactor":5,"legendFormat":"{{cpu}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node_load1{job=\"integrations/node_exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"1m load average","refId":"A"},{"expr":"node_load5{job=\"integrations/node_exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"5m load average","refId":"B"},{"expr":"node_load15{job=\"integrations/node_exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"15m load average","refId":"C"},{"expr":"count(node_cpu_seconds_total{job=\"integrations/node_exporter\", instance=\"$instance\", mode=\"idle\"})","format":"time_series","intervalFactor":2,"legendFormat":"logical cores","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Load Average","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n node_memory_MemTotal_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"node_memory_Buffers_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory buffers","refId":"B"},{"expr":"node_memory_Cached_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"C"},{"expr":"node_memory_MemFree_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"}},"gridPos":{},"id":5,"span":3,"targets":[{"expr":"100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"integrations/node_exporter\", instance=\"$instance\"})\n* 100\n)\n","format":"time_series","intervalFactor":2,"legendFormat":""}],"title":"Memory Usage","transparent":false,"type":"gauge"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ read| written/","yaxis":1},{"alias":"/ io time/","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_disk_read_bytes_total{job=\"integrations/node_exporter\", instance=\"$instance\", device!=\"\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} read","refId":"A"},{"expr":"rate(node_disk_written_bytes_total{job=\"integrations/node_exporter\", instance=\"$instance\", device!=\"\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} written","refId":"B"},{"expr":"rate(node_disk_io_time_seconds_total{job=\"integrations/node_exporter\", instance=\"$instance\", device!=\"\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{},"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byName","options":"Mounted on"},"properties":[{"id":"custom.width","value":260}]},{"matcher":{"id":"byName","options":"Size"},"properties":[{"id":"custom.width","value":93}]},{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":72}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"custom.width","value":88}]},{"matcher":{"id":"byName","options":"Used, %"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.displayMode","value":"gradient-gauge"},{"id":"max","value":1},{"id":"min","value":0}]}]},"gridPos":{},"id":7,"span":6,"targets":[{"expr":"max by (mountpoint) (node_filesystem_size_bytes{job=\"integrations/node_exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""},{"expr":"max by (mountpoint) (node_filesystem_avail_bytes{job=\"integrations/node_exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""}],"title":"Disk Space Usage","transformations":[{"id":"groupBy","options":{"fields":{"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #B":{"aggregations":["lastNotNull"],"operation":"aggregate"},"mountpoint":{"aggregations":[],"operation":"groupby"}}}},{"id":"merge","options":{}},{"id":"calculateField","options":{"alias":"Used","binary":{"left":"Value #A (lastNotNull)","operator":"-","reducer":"sum","right":"Value #B (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"calculateField","options":{"alias":"Used, %","binary":{"left":"Used","operator":"/","reducer":"sum","right":"Value #A (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Value #A (lastNotNull)":"Size","Value #B (lastNotNull)":"Available","mountpoint":"Mounted on"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Mounted on"}]}}],"transparent":false,"type":"table"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network received (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_receive_bytes_total{job=\"integrations/node_exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Received","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network transmitted (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_transmit_bytes_total{job=\"integrations/node_exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Transmitted","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter","nodes","hardware","cpu","ram"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(node_uname_info{job=\"integrations/node_exporter\", sysname!=\"Darwin\"}, instance)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"utc","title":"Node Exporter Nomadder1 -Nodes","uid":"fce5a965-b51b-4f98-818b-082850a33bdf","version":1} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Cluster.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Cluster.json new file mode 100644 index 0000000..4c58d66 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Cluster.json @@ -0,0 +1 @@ +{"annotations":{"list":[]},"editable":false,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":43,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"((\n instance:node_cpu_utilisation:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}))\n","format":"time_series","intervalFactor":2,"legendFormat":"{{ instance }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance:node_load1_per_cpu:ratio{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Saturation (Load1 per CPU)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance:node_memory_utilisation:ratio{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_vmstat_pgmajfault:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Major Page Faults)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/Receive/","stack":"A"},{"alias":"/Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Receive","refId":"A"},{"expr":"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Utilisation (Bytes Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ Receive/","stack":"A"},{"alias":"/ Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_drop_excluding_lo:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Receive","refId":"A"},{"expr":"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Saturation (Drops Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Saturation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk IO","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"integrations/node_exporter\", fstype!=\"\", mountpoint!=\"\", nomadder1=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"integrations/node_exporter\", fstype!=\"\", mountpoint!=\"\", nomadder1=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"integrations/node_exporter\", fstype!=\"\", mountpoint!=\"\", nomadder1=\"$cluster\"})))\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Space Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk Space","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter","nodes","hardware","cpu","ram"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(node_time_seconds, nomadder1)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"utc","title":"Node Exporter Nomadder1 -USE Method / Cluster","uid":"d5db4998-72ae-4ece-924a-1a2f431bacf8","version":1} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Node.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Node.json new file mode 100644 index 0000000..b524959 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Node Exporter Nomadder1 -USE Method Node.json @@ -0,0 +1 @@ +{"annotations":{"list":[]},"editable":false,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":44,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_cpu_utilisation:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Utilisation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_load1_per_cpu:ratio{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Saturation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Saturation (Load1 per CPU)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_memory_utilisation:ratio{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Utilisation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_vmstat_pgmajfault:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Major page Faults","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Major Page Faults)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/Receive/","stack":"A"},{"alias":"/Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Receive","refId":"A"},{"expr":"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Utilisation (Bytes Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ Receive/","stack":"A"},{"alias":"/ Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_drop_excluding_lo:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Receive","refId":"A"},{"expr":"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Saturation (Drops Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance_device:node_disk_io_time_seconds:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"integrations/node_exporter\", instance=\"$instance\", nomadder1=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Saturation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk IO","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"integrations/node_exporter\", fstype!=\"\", instance=\"$instance\", nomadder1=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"integrations/node_exporter\", fstype!=\"\", instance=\"$instance\", nomadder1=\"$cluster\"})\n ) != 0\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Space Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk Space","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter","nodes","hardware","cpu","ram"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(node_time_seconds, nomadder1)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(node_exporter_build_info{job=\"integrations/node_exporter\", nomadder1=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"utc","title":"Node Exporter Nomadder1 -USE Method / Node","uid":"d37ff948-c993-4078-9205-2de44c06868a","version":1} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Nodes Overview.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Nodes Overview.json new file mode 100644 index 0000000..d1c3571 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/NodeExporter/Nodes Overview.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":59,"links":[],"liveNow":false,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":12,"panels":[],"title":"Node Info","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"custom":{"align":"auto","cellOptions":{"type":"auto"},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Pretty name"},"properties":[{"id":"custom.width","value":372}]},{"matcher":{"id":"byName","options":"Kernel Version"},"properties":[{"id":"custom.width","value":290}]}]},"gridPos":{"h":8,"w":24,"x":0,"y":1},"id":2,"maxDataPoints":100,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true,"sortBy":[]},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"node_uname_info{job=\"integrations/node_exporter\",instance=~\"$node\"} * on(instance) group_left(pretty_name,name,version_codename,version_id) node_os_info{job=\"integrations/node_exporter\",instance=~\"$node\"}","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"Os overview","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"agent_hostname":true,"domainname":true,"instance":true,"job":true,"version":true,"version_id":true},"indexByName":{"Time":7,"Value":14,"agent_hostname":8,"domainname":9,"instance":10,"job":11,"machine":3,"name":1,"nodename":0,"pretty_name":4,"release":5,"sysname":2,"version":12,"version_codename":6,"version_id":13},"renameByName":{"Time":"","agent_hostname":"","instance":"","machine":"Arch","name":"Os Name","nodename":"Node Name","pretty_name":"Pretty name","release":"Kernel Version","sysname":"Os Type","version_codename":"Code Name"}}}],"transparent":true,"type":"table"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"mappings":[{"options":{"0":{"color":"dark-red","index":0,"text":"Offline"},"1":{"color":"dark-green","index":1,"text":"Online"}},"type":"value"}]},"overrides":[]},"gridPos":{"h":6,"w":24,"x":0,"y":9},"id":1,"options":{"autoSizeColumns":true,"autoSizePolygons":true,"autoSizeRows":true,"compositeConfig":{"animationSpeed":"1500","composites":[],"enabled":true},"ellipseCharacters":21,"ellipseEnabled":true,"globalAutoScaleFonts":true,"globalClickthrough":"","globalClickthroughNewTabEnabled":true,"globalClickthroughSanitizedEnabled":true,"globalDecimals":2,"globalDisplayMode":"all","globalDisplayTextTriggeredEmpty":"OK","globalFillColor":"rgba(10, 85, 161, 1)","globalFontSize":25,"globalGradientsEnabled":true,"globalOperator":"mean","globalPolygonBorderColor":"rgba(0, 0, 0, 0)","globalPolygonBorderSize":1,"globalPolygonSize":100,"globalRegexPattern":"","globalShape":"hexagon_pointed_top","globalShowTooltipColumnHeadersEnabled":true,"globalShowValueEnabled":true,"globalTextFontAutoColorEnabled":true,"globalTextFontColor":"#000000","globalTextFontFamily":"Roboto Mono","globalThresholdsConfig":[],"globalTooltipsEnabled":true,"globalTooltipsFontFamily":"Roboto","globalTooltipsShowTimestampEnabled":false,"globalUnitFormat":"short","layoutDisplayLimit":100,"layoutNumColumns":8,"layoutNumRows":1,"overrideConfig":{"overrides":[]},"sortByDirection":1,"sortByField":"name","tooltipDisplayMode":"all","tooltipDisplayTextTriggeredEmpty":"OK","tooltipPrimarySortByField":"thresholdLevel","tooltipPrimarySortDirection":1,"tooltipSecondarySortByField":"value","tooltipSecondarySortDirection":1},"pluginVersion":"2.0.7","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"up{job=\"integrations/node_exporter\",instance=~\"$node\"}[1m]","instant":true,"legendFormat":"{{instance}}","range":false,"refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"1- max_over_time(up{job=\"integrations/node_exporter\",instance=~\"$node\"}[24h]) unless max_over_time(up{job=\"integrations/node_exporter\",instance=~\"$node\"}[1m])","hide":false,"instant":true,"legendFormat":"{{instance}}","range":false,"refId":"B"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"node_os_info{job=\"integrations/node_exporter\",instance=~\"$node\"} ","hide":true,"instant":true,"legendFormat":"{{instance}}","range":false,"refId":"C"}],"title":"Nodes Overview","transformations":[],"transparent":true,"type":"grafana-polystat-panel"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":15},"id":9,"panels":[],"title":"CPU","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"light-blue","value":null},{"color":"#EAB839","value":70},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":16},"hideTimeOverride":false,"id":3,"links":[],"maxPerRow":6,"options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"node","repeatDirection":"h","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"sum by (instance) (1- irate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\"}[5m])) / sum by (instance) (irate(node_cpu_seconds_total{instance=\"$node\"}[5m])) * 100","format":"time_series","hide":false,"intervalFactor":1,"range":true,"refId":"B","step":240}],"title":"CPU Used $node","transparent":true,"type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":19},"id":10,"panels":[],"title":"RAM","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Non available RAM memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"light-blue","value":null},{"color":"#EAB839","value":70},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":20},"hideTimeOverride":false,"id":8,"links":[],"maxPerRow":6,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"node","repeatDirection":"h","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"integrations/node_exporter\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"integrations/node_exporter\"})","format":"time_series","hide":false,"intervalFactor":1,"range":true,"refId":"B","step":240}],"title":"RAM Used $node","transparent":true,"type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":11,"panels":[],"title":"File System","type":"row"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Used Root FS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null},{"color":"#EAB839","value":70},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":24},"id":7,"links":[],"maxPerRow":6,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"node","repeatDirection":"h","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"integrations/node_exporter\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"integrations/node_exporter\",mountpoint=\"/\",fstype!=\"rootfs\"})","format":"time_series","intervalFactor":1,"range":true,"refId":"A","step":240}],"title":"Root FS Used $node","transparent":true,"type":"stat"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[{"current":{"selected":true,"text":["All"],"value":["$__all"]},"datasource":{"type":"prometheus","uid":"mimir"},"definition":"label_values(node_uname_info,instance)","hide":0,"includeAll":true,"label":"Host:","multi":true,"name":"node","options":[],"query":{"query":"label_values(node_uname_info,instance)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-6h","to":"now"},"timepicker":{},"timezone":"","title":"Nodes Overview","uid":"ed22277d-ffa2-4a08-97b8-a77687ec05cd","version":23,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Consul Overview.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Consul Overview.json new file mode 100644 index 0000000..284fd8e --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Consul Overview.json @@ -0,0 +1 @@ +{"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":55,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"colorBackground":true,"colors":["#d44a3a","rgba(237, 129, 40, 0.89)","#299c46"],"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"none","id":1,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"instance","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"consul_up{job=~\"$job\",instance=~\"$instance\"}","format":"time_series","instant":true,"intervalFactor":2,"refId":"A"}],"thresholds":"0.5,0.5","timeFrom":null,"timeShift":null,"title":"$instance","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","valueMaps":[{"op":"=","text":"DOWN","value":"0"},{"op":"=","text":"UP","value":"1"}],"xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Up","titleSize":"h6"},{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"colorBackground":true,"colors":["rgba(237, 129, 40, 0.89)","rgba(237, 129, 40, 0.89)","#299c46"],"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"none","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"instance","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"(rate(consul_raft_leader_lastcontact_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) \u003e bool 0)\n or\n(consul_up{job=~\"$job\",instance=~\"$instance\"} == bool 0)\n","format":"time_series","instant":true,"intervalFactor":2,"refId":"A"}],"thresholds":"0.5,0.5","timeFrom":null,"timeShift":null,"title":"$instance","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","valueMaps":[{"op":"=","text":"FOLLOWER","value":"0"},{"op":"=","text":"LEADER","value":"1"}],"xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Leader","titleSize":"h6"},{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"colorBackground":true,"colors":["#d44a3a","rgba(237, 129, 40, 0.89)","#299c46"],"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"none","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"instance","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"consul_raft_leader{job=~\"$job\",instance=~\"$instance\"}","format":"time_series","instant":true,"intervalFactor":2,"refId":"A"}],"thresholds":"0.5,0.5","timeFrom":null,"timeShift":null,"title":"$instance","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","valueMaps":[{"op":"=","text":"NO LEADER","value":"0"},{"op":"=","text":"HAS LEADER","value":"1"}],"xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Has Leader","titleSize":"h6"},{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"colorBackground":true,"colors":["#d44a3a","rgba(237, 129, 40, 0.89)","#299c46"],"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"none","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"instance","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"consul_raft_peers{job=~\"$job\",instance=~\"$instance\"}","format":"time_series","instant":true,"intervalFactor":2,"refId":"A"}],"thresholds":"1,2","timeFrom":null,"timeShift":null,"title":"$instance","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"# Peers","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(consul_http_request_count{job=~\"$job\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","legendLink":null,"step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"QPS","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(consul_http_request{job=~\"$job\", quantile=\"0.99\"}) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"99th Percentile","legendLink":"{{instance}}","step":10},{"expr":"max(consul_http_request{job=~\"$job\", quantile=\"0.5\"}) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"50th Percentile","legendLink":"{{instance}}","step":10},{"expr":"sum(rate(consul_http_request{job=~\"$job\"}[$__rate_interval])) by (instance) / sum(rate(consul_http_request{job=~\"$job\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"Average","legendLink":null,"step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Latency","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Consul Server","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":[],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":0,"includeAll":true,"label":"job","multi":true,"name":"job","options":[],"query":"label_values(consul_up, job)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":0,"includeAll":true,"label":"instance","multi":true,"name":"instance","options":[],"query":"label_values(consul_up{job=~\"$job\"}, instance)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"utc","title":"Consul Overview","uid":"f71ad233ae18512edec190eee2558346","version":1} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad Cluster overview.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad Cluster overview.json new file mode 100644 index 0000000..b1baf89 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad Cluster overview.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Nomad cluster dashboard. Using built in nomad's exporter and loki. Example cluster is present there https://github.com/theztd/startup-infra-docker in ","editable":true,"fiscalYearStartMonth":0,"gnetId":17549,"graphTooltip":1,"id":1,"links":[],"liveNow":false,"panels":[{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":207,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"fixedColor":"green","mode":"fixed"},"decimals":0,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"text","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":0,"y":1},"id":221,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_status_running{}","instant":true,"interval":"","legendFormat":"Running","refId":"A"}],"type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"In the whole cluster","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"unit":"none"},"overrides":[]},"gridPos":{"h":6,"w":6,"x":2,"y":1},"id":107,"options":{"displayLabels":["percent"],"legend":{"displayMode":"list","placement":"right","showLegend":true,"values":[]},"pieType":"donut","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.0.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"sum(nomad_client_allocs_cpu_allocated) by (exported_job)","interval":"","legendFormat":"{{exported_job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"sum(nomad_client_unallocated_cpu)","hide":false,"legendFormat":"Free cluster CPU","range":true,"refId":"B"}],"title":"Allocated CPU ($datacenter)","type":"piechart"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"inspect":false},"links":[{"title":"Set time range","url":"https://mon.cinarra.com/d/4NYOvS-7z/nomad?orgId=1\u0026var-env=$env\u0026var-alloc_id=${__data.fields.alloc_id}\u0026from=${__data.fields.from}\u0026to=${__data.fields.to}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"from"},"properties":[{"id":"custom.hidden","value":true}]},{"matcher":{"id":"byName","options":"to"},"properties":[{"id":"custom.hidden","value":true}]},{"matcher":{"id":"byName","options":"alloc_id"},"properties":[{"id":"custom.width","value":287}]},{"matcher":{"id":"byName","options":"Kills"},"properties":[{"id":"custom.width","value":45}]},{"matcher":{"id":"byName","options":"Instance"},"properties":[{"id":"custom.width","value":131}]},{"matcher":{"id":"byName","options":"task"},"properties":[{"id":"custom.width","value":92}]}]},"gridPos":{"h":4,"w":13,"x":8,"y":1},"id":215,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"frameIndex":0,"showHeader":true,"sortBy":[]},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":true,"expr":"nomad_client_allocs_oom_killed{}","format":"table","instant":false,"interval":"","intervalFactor":1,"legendFormat":"oom","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":true,"expr":"timestamp(nomad_client_allocs_oom_killed{}) * 1000 - 3600000","format":"table","hide":false,"instant":false,"interval":"","intervalFactor":1,"legendFormat":"from","refId":"B"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":true,"expr":"timestamp(nomad_client_allocs_oom_killed{}) * 1000 + 3600000","format":"table","hide":false,"instant":false,"interval":"","intervalFactor":1,"legendFormat":"to","refId":"C"}],"title":"OOM kills","transformations":[{"id":"seriesToColumns","options":{"byField":"alloc_id"}},{"id":"filterFieldsByName","options":{"include":{"pattern":"alloc_id|exporter_job 1|Time 1|instance 1|namespace 1|task 1|task_group 1|Value .*"}}},{"id":"organize","options":{"excludeByName":{"Value #B":true,"Value #C":true,"namespace 1":true,"task_group 1":true},"indexByName":{"Time 1":0,"Value #A":6,"Value #B":7,"Value #C":8,"alloc_id":3,"instance 1":1,"namespace 1":4,"task 1":2,"task_group 1":5},"renameByName":{"Time 1":"Time","Value #A":"Kills","Value #B":"from","Value #C":"to","alloc_id":"","instance 1":"Instance","namespace 1":"namespace","task 1":"task","task_group 1":"group"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Time"}]}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"0":{"color":"red","index":1,"text":"Unhealthy"},"1":{"color":"green","index":0,"text":"Healthy"}},"type":"value"}],"thresholds":{"mode":"absolute","steps":[{"color":"text","value":null}]}},"overrides":[]},"gridPos":{"h":12,"w":3,"x":21,"y":1},"id":229,"options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"up{job=\"integrations/nomad\"}","format":"time_series","instant":true,"interval":"","legendFormat":"{{instance}}","refId":"A"}],"title":"Node status","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"fixedColor":"light-orange","mode":"fixed"},"decimals":0,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"text","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":0,"y":4},"id":217,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_status_pending","instant":true,"interval":"","legendFormat":"Pending","refId":"A"}],"type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Run"},"properties":[{"id":"custom.cellOptions","value":{"mode":"basic","type":"color-background"}},{"id":"color","value":{"fixedColor":"green","mode":"fixed"}},{"id":"custom.width","value":64}]},{"matcher":{"id":"byName","options":"Complete"},"properties":[{"id":"custom.cellOptions","value":{"mode":"basic","type":"color-background"}},{"id":"color","value":{"fixedColor":"semi-dark-green","mode":"fixed"}},{"id":"custom.width","value":65}]},{"matcher":{"id":"byName","options":"Failed"},"properties":[{"id":"custom.cellOptions","value":{"mode":"basic","type":"color-background"}},{"id":"color","value":{"fixedColor":"semi-dark-red","mode":"fixed"}},{"id":"custom.width","value":63}]},{"matcher":{"id":"byName","options":"Starting"},"properties":[{"id":"custom.cellOptions","value":{"mode":"basic","type":"color-background"}},{"id":"color","value":{"fixedColor":"text","mode":"fixed"}},{"id":"custom.width","value":89}]},{"matcher":{"id":"byName","options":"Quequed"},"properties":[{"id":"custom.cellOptions","value":{"mode":"basic","type":"color-background"}},{"id":"color","value":{"fixedColor":"transparent","mode":"fixed"}},{"id":"custom.width","value":94}]},{"matcher":{"id":"byName","options":"Lost"},"properties":[{"id":"custom.cellOptions","value":{"mode":"basic","type":"color-background"}},{"id":"color","value":{"fixedColor":"#000000","mode":"fixed"}},{"id":"custom.width","value":62}]},{"matcher":{"id":"byName","options":"job"},"properties":[{"id":"custom.width","value":264}]},{"matcher":{"id":"byName","options":"Node"},"properties":[{"id":"custom.width","value":187}]},{"matcher":{"id":"byName","options":"Job"},"properties":[{"id":"custom.width","value":351}]}]},"gridPos":{"h":8,"w":13,"x":8,"y":5},"id":231,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":["Value #F","Value #A","Value #B","Value #D","Value #E","Value #C"],"reducer":["sum"],"show":true},"showHeader":true,"sortBy":[]},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_summary_running{}","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"running","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_summary_failed{}","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"failed","refId":"B"},{"datasource":{"type":"prometheus","uid":"mimir"},"exemplar":false,"expr":"nomad_nomad_job_summary_lost{}","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"lost","refId":"C"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_summary_starting{}","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"starting","refId":"D"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_summary_queued{}","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"queued","refId":"E"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_summary_complete{}","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"complete","range":false,"refId":"F"}],"title":"Allocations","transformations":[{"id":"seriesToColumns","options":{"byField":"task_group"}},{"id":"filterFieldsByName","options":{"include":{"names":["task_group","Value #A","Value #B","Value #C","host 1"],"pattern":"exported_job 1|namespace 1|task_group|Value .*"}}},{"id":"organize","options":{"excludeByName":{"exported_job 1":false,"host":true,"host 1":true,"namespace 1":true,"task_group":false},"indexByName":{"Value #A":4,"Value #B":5,"Value #C":6,"exported_job 1":0,"host 1":1,"namespace 1":3,"task_group":2},"renameByName":{"Value #A":"Run","Value #B":"Failed","Value #C":"Lost","Value #D":"Starting","Value #E":"Quequed","Value #F":"Complete","exported_job 1":"Job","host 1":"Node","namespace 1":"namespace","task_group":"group","task_group 1":"Group"}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"fixedColor":"super-light-red","mode":"fixed"},"decimals":0,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"text","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":0,"y":7},"id":219,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_nomad_job_status_dead","instant":true,"interval":"","legendFormat":"Dead","refId":"A"}],"type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"In whole cluster","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"unit":"bytes"},"overrides":[]},"gridPos":{"h":6,"w":6,"x":2,"y":7},"id":86,"options":{"displayLabels":["percent"],"legend":{"displayMode":"list","placement":"right","showLegend":true,"values":[]},"pieType":"donut","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.0.6","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"sum(nomad_client_allocs_memory_allocated) by (exported_job)","interval":"","legendFormat":"{{exported_job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"sum(nomad_client_host_memory_free)","hide":false,"legendFormat":"Free cluster MEM","range":true,"refId":"B"}],"title":"Allocated MEM ($datacenter)","type":"piechart"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"fixedColor":"semi-dark-red","mode":"fixed"},"decimals":0,"mappings":[],"max":100,"min":0,"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"text","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":0,"y":10},"id":248,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"9.5.2","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":false,"expr":"nomad_client_allocs_oom_killed","instant":true,"interval":"","legendFormat":"Killed","range":false,"refId":"A"}],"transformations":[],"type":"stat"}],"title":"Cluster Summary","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"000000001"},"gridPos":{"h":1,"w":24,"x":0,"y":1},"id":2,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"displayName":"($host)","mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"light-red","value":null},{"color":"light-orange","value":360},{"color":"light-green","value":86400}]},"unit":"dtdurations"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":0,"y":2},"id":4,"links":[],"maxDataPoints":100,"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"nomad_client_uptime{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","intervalFactor":1,"refId":"A"}],"title":"Uptime","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"displayName":"$host","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":3,"y":2},"id":11,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"nomad_client_host_memory_free{datacenter=~\"$datacenter\", host=~\"$host\"} /nomad_client_host_memory_total{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Total","range":true,"refId":"B"}],"title":"Free memmory","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"light-red","value":null},{"color":"light-orange","value":15},{"color":"light-green","value":30}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":3,"w":5,"x":9,"y":2},"id":13,"links":[],"options":{"displayMode":"lcd","minVizHeight":0,"minVizWidth":0,"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"valueMode":"color"},"pluginVersion":"9.5.2","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\" ,disk=~\"/dev/sd.*\"}) by (disk) / max(nomad_client_host_disk_size{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk) ","format":"time_series","intervalFactor":1,"legendFormat":"{{disk}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk)","format":"time_series","hide":true,"intervalFactor":1,"legendFormat":"Available - {{disk}}","range":true,"refId":"A"}],"title":"Free Disk Space ($host)","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":3,"w":4,"x":14,"y":2},"id":58,"links":[],"options":{"displayMode":"lcd","minVizHeight":2,"minVizWidth":0,"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"valueMode":"color"},"pluginVersion":"9.5.2","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"nomad_client_allocations_migrating{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","intervalFactor":1,"legendFormat":"Migrating","refId":"A"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"nomad_client_allocations_blocked{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","intervalFactor":1,"legendFormat":"Blocked","refId":"B"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"nomad_client_allocations_pending{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","intervalFactor":1,"legendFormat":"Pending","refId":"C"},{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"nomad_client_allocations_running{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","intervalFactor":1,"legendFormat":"Running","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"nomad_client_allocations_terminal{datacenter=~\"$datacenter\", host=~\"$host\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Terminal","refId":"E"}],"title":"Allocation status","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"displayName":"$host","mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"light-red","value":null},{"color":"#EAB839","value":10},{"color":"light-green","value":50}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":18,"y":2},"id":33,"links":[],"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","expr":"nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"})","format":"time_series","intervalFactor":1,"range":true,"refId":"A"}],"title":"Unalocated CPU","type":"stat"},{"datasource":{"type":"prometheus","uid":"mimir"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"displayName":"$host","mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"light-red","value":null},{"color":"#EAB839","value":10},{"color":"light-green","value":50}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":3,"w":3,"x":21,"y":2},"id":40,"links":[],"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"9.5.2","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"expr":"nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"})","format":"time_series","intervalFactor":1,"refId":"A"}],"title":"Unalocated Mem","type":"stat"}],"targets":[{"datasource":{"type":"prometheus","uid":"000000001"},"refId":"A"}],"title":"Node Status","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"id":247,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Time difference between when a message was first passed into Raft and when the resulting commit took place on the leader","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":100},{"color":"red","value":500}]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":15},"id":223,"options":{"legend":{"calcs":["mean","max","lastNotNull"],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.1.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"exemplar":true,"expr":"nomad_raft_commitTime{quantile=\"0.9\"}","interval":"","intervalFactor":1,"legendFormat":"{{ instance }}","refId":"A"}],"title":"Raft Commit Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"mimir"},"description":"Number of Raft transactions - general indicator of the write load on the servers.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":30,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"stepBefore","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":15},"id":225,"options":{"legend":{"calcs":["mean","max","lastNotNull"],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.1.0","targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"editorMode":"code","exemplar":true,"expr":"increase(nomad_raft_apply{}[$__rate_interval])","interval":"","legendFormat":"{{ instance }}","range":true,"refId":"A"}],"title":"Raft Apply Count","type":"timeseries"}],"title":"RAFT status","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"id":239,"panels":[{"datasource":{"type":"loki","uid":"loki"},"description":"","gridPos":{"h":13,"w":24,"x":0,"y":16},"id":127,"options":{"dedupStrategy":"none","enableLogDetails":true,"prettifyLogMessage":false,"showCommonLabels":false,"showLabels":false,"showTime":true,"sortOrder":"Descending","wrapLogMessage":true},"targets":[{"datasource":{"type":"loki","uid":"loki"},"editorMode":"code","expr":"{job=\"nomad.service\"} | json | line_format `{{.service_name}}@{{.service_node_name}} [{{.log_levelEmoji}} {{.level}}] [{{.tags}}] {{.message}}`","hide":false,"queryType":"range","refId":"A"}],"title":"Logs ($datacenter)","type":"logs"}],"title":"Logs","type":"row"}],"refresh":"5s","schemaVersion":38,"style":"dark","tags":["nomad","host","cluster"],"templating":{"list":[{"current":{"selected":false,"text":"nomadder1","value":"nomadder1"},"datasource":{"type":"prometheus","uid":"mimir"},"definition":"","hide":0,"includeAll":false,"label":"DC","multi":false,"name":"datacenter","options":[],"query":{"query":"label_values(nomad_client_uptime, datacenter)","refId":"de1-datacenter-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"mimir"},"definition":"","hide":0,"includeAll":true,"label":"Host","multi":true,"name":"host","options":[],"query":{"query":"label_values(nomad_client_uptime{datacenter=~\"$datacenter\"}, host)","refId":"de1-host-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-15m","to":"now"},"timepicker":{"refresh_intervals":["1s","5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Nomad Cluster overview","uid":"masiNomadCluster","version":4,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad cluster.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad cluster.json new file mode 100644 index 0000000..9983368 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad cluster.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"id":53,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"grafanacloud-prom"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":24,"targets":[{"datasource":{"type":"prometheus","uid":"grafanacloud-prom"},"refId":"A"}],"title":"Allocations","type":"row"},{"datasource":{"uid":"$datasource"},"description":"CPU allocated on $instance","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":80},{"color":"red","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":5,"w":5,"x":0,"y":1},"id":33,"links":[],"maxDataPoints":100,"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"text":{}},"pluginVersion":"9.3.2","repeat":"instance","repeatDirection":"v","targets":[{"datasource":{"uid":"$datasource"},"expr":"nomad_client_allocated_cpu{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}/(nomad_client_unallocated_cpu{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}+nomad_client_allocated_cpu{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})*100","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"CPU allocated","type":"gauge"},{"datasource":{"uid":"$datasource"},"description":"Memory allocated on $instance","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":80},{"color":"red","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":5,"w":5,"x":5,"y":1},"id":40,"links":[],"maxDataPoints":100,"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"text":{}},"pluginVersion":"9.3.2","repeat":"instance","repeatDirection":"v","targets":[{"datasource":{"uid":"$datasource"},"expr":"nomad_client_allocated_memory{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}/(nomad_client_unallocated_memory{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}+nomad_client_allocated_memory{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})*100","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Memory allocated","type":"gauge"},{"datasource":{"uid":"$datasource"},"description":"Disk allocated on $instance","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":2,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":80},{"color":"red","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":5,"w":5,"x":10,"y":1},"id":48,"links":[],"maxDataPoints":100,"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"text":{}},"pluginVersion":"9.3.2","repeat":"instance","repeatDirection":"v","targets":[{"datasource":{"uid":"$datasource"},"expr":"nomad_client_allocated_disk{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}/(nomad_client_unallocated_disk{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}+nomad_client_allocated_disk{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})*100","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Disk allocated","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"custom":{"fillOpacity":70,"lineWidth":1,"spanNulls":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"allocs"},"overrides":[]},"gridPos":{"h":5,"w":9,"x":15,"y":1},"id":58,"links":[],"options":{"alignValue":"left","legend":{"displayMode":"list","placement":"bottom","showLegend":false},"mergeValues":true,"rowHeight":0.95,"showValue":"auto","tooltip":{"mode":"single","sort":"none"}},"repeat":"instance","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum by (datacenter) (nomad_client_allocations_migrating{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Migrating","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum by (datacenter) (nomad_client_allocations_blocked{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Blocked","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum by (datacenter) (nomad_client_allocations_pending{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Pending","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum by (datacenter) (nomad_client_allocations_running{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Running","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum by (datacenter) (nomad_client_allocations_terminal{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Terminal","refId":"E"}],"title":"Summary","type":"state-timeline"},{"datasource":{"type":"prometheus","uid":"grafanacloud-prom"},"gridPos":{"h":1,"w":24,"x":0,"y":6},"id":2,"targets":[{"datasource":{"type":"prometheus","uid":"grafanacloud-prom"},"refId":"A"}],"title":"Nomad clients","type":"row"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"dtdurations"},"overrides":[]},"gridPos":{"h":5,"w":4,"x":0,"y":7},"id":4,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"9.3.2","repeat":"instance","repeatDirection":"v","targets":[{"datasource":{"uid":"$datasource"},"expr":"nomad_client_uptime{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Uptime","type":"stat"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":80},{"color":"red","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":5,"w":5,"x":4,"y":7},"id":7,"links":[],"maxDataPoints":100,"options":{"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"text":{}},"pluginVersion":"9.3.2","repeat":"instance","repeatDirection":"v","targets":[{"datasource":{"uid":"$datasource"},"expr":"100-sum(nomad_client_host_cpu_idle{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}) / count(nomad_client_host_cpu_idle{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"})","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"CPU usage","type":"gauge"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":5,"w":5,"x":9,"y":7},"hiddenSeries":false,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.2","pointradius":5,"points":false,"renderer":"flot","repeat":"instance","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"uid":"$datasource"},"expr":"nomad_client_host_memory_total{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Total","refId":"B"},{"datasource":{"uid":"$datasource"},"expr":"nomad_client_host_memory_free{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"Free","refId":"A"},{"datasource":{"uid":"$datasource"},"expr":"nomad_client_host_memory_used{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Used","refId":"C"},{"datasource":{"uid":"$datasource"},"expr":"nomad_client_host_memory_available{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Available","refId":"D"}],"thresholds":[],"timeRegions":[],"title":"Memory","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"decbytes","logBase":1,"show":true},{"format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":5,"w":5,"x":14,"y":7},"hiddenSeries":false,"id":13,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.2","pointradius":5,"points":false,"renderer":"flot","repeat":"instance","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"uid":"$datasource"},"expr":"max(nomad_client_host_disk_size{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}) by (disk)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Total - {{disk}}","refId":"B"},{"datasource":{"uid":"$datasource"},"expr":"avg(nomad_client_host_disk_available{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}) by (disk)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Available - {{disk}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Disk usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":0,"format":"decbytes","logBase":1,"show":true},{"format":"decbytes","logBase":1,"show":false}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":5,"w":5,"x":19,"y":7},"hiddenSeries":false,"id":15,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.3.2","pointradius":5,"points":false,"renderer":"flot","repeat":"instance","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"uid":"$datasource"},"expr":"avg(nomad_client_host_disk_inodes_percent{job=~\"$job\", datacenter=~\"$datacenter\", instance=~\"$instance\"}) by (disk)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{disk}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Disk inodes","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"percent","logBase":1,"show":true},{"format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"refresh":"30s","schemaVersion":37,"style":"dark","tags":["nomad-integration"],"templating":{"list":[{"current":{"selected":false,"text":"grafanacloud-k3d-prom","value":"grafanacloud-k3d-prom"},"hide":0,"includeAll":false,"label":"Data Source","multi":false,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"(?!grafanacloud-usage|grafanacloud-ml-metrics).+","skipUrlSync":false,"type":"datasource"},{"allValue":".+","current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(nomad_client_uptime, job)","hide":0,"includeAll":true,"label":"job","multi":true,"name":"job","options":[],"query":{"query":"label_values(nomad_client_uptime, job)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"type":"query"},{"current":{"isNone":true,"selected":false,"text":"None","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(nomad_client_uptime, datacenter)","hide":0,"includeAll":false,"label":"DC","multi":false,"name":"datacenter","options":[],"query":{"query":"label_values(nomad_client_uptime, datacenter)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(nomad_client_uptime{datacenter=\"$datacenter\"}, instance)","hide":0,"includeAll":true,"label":"instance","multi":true,"name":"instance","options":[],"query":{"query":"label_values(nomad_client_uptime{datacenter=\"$datacenter\"}, instance)","refId":"prometheus-instance-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Nomad cluster","uid":"CiP3mZVik","version":1,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad jobs.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad jobs.json new file mode 100644 index 0000000..e53d53f --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/Nomad Cluster Components/Nomad jobs.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Nomad jobs metrics","editable":true,"fiscalYearStartMonth":0,"gnetId":6281,"graphTooltip":0,"id":54,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"grafanacloud-prom"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":9,"panels":[],"repeat":"instance","targets":[{"datasource":{"type":"prometheus","uid":"grafanacloud-prom"},"refId":"A"}],"title":"$instance","type":"row"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":0,"y":1},"id":2,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.4.7","targets":[{"datasource":{"uid":"$datasource"},"expr":"avg(nomad_client_allocs_cpu_total_percent{job=~\"$job\", instance=~\"$instance\"}) by(exported_job, task)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{task}}","refId":"A"}],"title":"CPU usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"timeticks"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":12,"y":1},"id":3,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.4.7","targets":[{"datasource":{"type":"prometheus","uid":"1_UFfQJGk"},"exemplar":true,"expr":"avg(nomad_client_allocs_cpu_total_ticks{job=~\"$job\", instance=~\"$instance\"}) by (exported_job, task)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{task}}","refId":"A"}],"title":"CPU total ticks","type":"timeseries"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":0,"y":7},"id":6,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.4.7","targets":[{"datasource":{"uid":"$datasource"},"expr":"avg(nomad_client_allocs_memory_rss{job=~\"$job\", instance=~\"$instance\"}) by(exported_job, task)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{task}}","refId":"A"}],"title":"RSS","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":12,"y":7},"id":7,"links":[],"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.4.7","targets":[{"datasource":{"type":"prometheus","uid":"1_UFfQJGk"},"exemplar":true,"expr":"avg(nomad_client_allocs_memory_cache{job=~\"$job\", instance=~\"$instance\"}) by (exported_job, task)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{task}}","refId":"A"}],"title":"Memory cache","type":"timeseries"}],"refresh":"30s","schemaVersion":37,"style":"dark","tags":["nomad-integration"],"templating":{"list":[{"allValue":".+","current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(nomad_client_uptime, job)","hide":0,"includeAll":true,"label":"job","multi":true,"name":"job","options":[],"query":{"query":"label_values(nomad_client_uptime, job)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"type":"query"},{"current":{"selected":false,"text":"grafanacloud-k3d-prom","value":"grafanacloud-k3d-prom"},"hide":0,"includeAll":false,"label":"Data Source","multi":false,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"(?!grafanacloud-usage|grafanacloud-ml-metrics).+","skipUrlSync":false,"type":"datasource"},{"current":{"isNone":true,"selected":false,"text":"None","value":""},"datasource":{"uid":"$datasource"},"definition":"label_values(nomad_client_uptime, datacenter)","hide":0,"includeAll":false,"label":"DC","multi":false,"name":"datacenter","options":[],"query":{"query":"label_values(nomad_client_uptime, datacenter)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":["All"],"value":["$__all"]},"datasource":{"uid":"$datasource"},"definition":"","hide":0,"includeAll":true,"label":"instance","multi":true,"name":"instance","options":[],"query":{"query":"label_values(nomad_client_uptime{datacenter=~\"$datacenter\"}, instance)","refId":"prometheus-instance-Variable-Query"},"refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Nomad jobs","uid":"TvqbbhViz","version":1,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/demo/Home.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/demo/Home.json new file mode 100644 index 0000000..fc4aaf9 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/dashboards/demo/Home.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":26,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":3,"w":24,"x":0,"y":0},"id":1,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"type":"welcome"},{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":9,"w":24,"x":0,"y":3},"id":123123,"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"type":"gettingstarted"},{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":15,"w":12,"x":0,"y":12},"id":3,"links":[],"options":{"folderId":0,"maxItems":30,"query":"","showHeadings":true,"showRecentlyViewed":true,"showSearch":false,"showStarred":true,"tags":[]},"pluginVersion":"9.5.2","tags":[],"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Dashboards","type":"dashlist"},{"datasource":{"type":"prometheus","uid":"mimir"},"gridPos":{"h":15,"w":12,"x":12,"y":12},"id":4,"links":[],"options":{"feedUrl":"https://grafana.com/blog/news.xml","showImage":true},"targets":[{"datasource":{"type":"prometheus","uid":"mimir"},"refId":"A"}],"title":"Latest from the blog","type":"news"}],"refresh":false,"schemaVersion":38,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-6h","to":"now"},"timepicker":{"hidden":true,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"],"type":"timepicker"},"timezone":"browser","title":"Home","uid":"c420593c-2cd6-4341-9104-3b40159f5c1d","version":2,"weekStart":""} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/AlertmanagerMimir.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/AlertmanagerMimir.json new file mode 100644 index 0000000..c4ef645 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/AlertmanagerMimir.json @@ -0,0 +1 @@ +{"access":"proxy","id":2,"jsonData":{"httpHeaderName1":"X-Scope-OrgID","implementation":"prometheus"},"name":"Alertmanager/Mimir","orgId":1,"type":"alertmanager","typeLogoUrl":"public/app/plugins/datasource/alertmanager/img/logo.svg","uid":"alertmanager","url":"http://mimir.service.consul:9009/alertmanager"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Grafana HTTP API.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Grafana HTTP API.json new file mode 100644 index 0000000..4d76314 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Grafana HTTP API.json @@ -0,0 +1 @@ +{"access":"proxy","id":9,"jsonData":{"url":"http://localhost:3000"},"name":"Grafana HTTP API","orgId":1,"type":"volkovlabs-grapi-datasource","typeLogoUrl":"public/plugins/volkovlabs-grapi-datasource/img/logo.svg","uid":"c81a45cb-f2ab-4e55-a09d-6a0d623bf8f0"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Loki.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Loki.json new file mode 100644 index 0000000..9808e19 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Loki.json @@ -0,0 +1 @@ +{"access":"proxy","id":3,"jsonData":{"alertmanagerUid":"alertmanager","derivedFields":[{"datasourceUid":"tempo","matcherRegex":"traceID=(\\w+)","name":"TraceID","url":"${__value.raw}"}],"httpHeaderName1":"X-Scope-OrgID","maxLines":2100},"name":"Loki","orgId":1,"type":"loki","typeLogoUrl":"public/app/plugins/datasource/loki/img/loki_icon.svg","uid":"loki","url":"http://loki.service.consul:3100"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Mimir.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Mimir.json new file mode 100644 index 0000000..3da2420 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Mimir.json @@ -0,0 +1 @@ +{"access":"proxy","id":1,"isDefault":true,"jsonData":{"alertmanagerUid":"alertmanager","derivedFields":[{"datasourceUid":"tempo","matcherRegex":"(?:traceID|trace_id)=(\\w+)","name":"TraceID","url":"${__value.raw}"}],"httpHeaderName1":"X-Scope-OrgID","httpMethod":"POST","manageAlerts":"true","prometheusType":"Mimir","prometheusVersion":"2.8.0"},"name":"Mimir","orgId":1,"type":"prometheus","typeLogoUrl":"public/app/plugins/datasource/prometheus/img/prometheus_logo.svg","uid":"mimir","url":"http://mimir.service.consul:9009/prometheus"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/OPC UA (Unified Architecture).json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/OPC UA (Unified Architecture).json new file mode 100644 index 0000000..4bb6541 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/OPC UA (Unified Architecture).json @@ -0,0 +1 @@ +{"access":"proxy","id":10,"jsonData":{},"name":"OPC UA (Unified Architecture)","orgId":1,"type":"grafana-opcua-datasource","typeLogoUrl":"public/plugins/grafana-opcua-datasource/img/opc_logo.png","uid":"b35a5f96-102a-4f86-9a72-b0d02e679e8b"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Tempo.json b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Tempo.json new file mode 100644 index 0000000..b0a22c9 --- /dev/null +++ b/ansible/setup/stack_observability/templates/grafana/provisioning/exported/data-sources/Tempo.json @@ -0,0 +1 @@ +{"access":"proxy","id":4,"jsonData":{"httpMethod":"GET","lokiSearch":{"datasourceUid":"loki"},"nodeGraph":{"enabled":true},"search":{"hide":false},"serviceMap":{"datasourceUid":"mimir"},"tracesToLogs":{"datasourceUid":"loki","filterBySpanID":false,"filterByTraceID":false,"mapTagNamesEnabled":false,"mappedTags":[{"key":"service.name","value":"service"}],"spanEndTimeShift":"1h","spanStartTimeShift":"1h","tags":["group","job","task","container","namespace"]},"tracesToMetrics":{"datasourceUid":"mimir","queries":[{"name":"Sample query","query":"sum(rate(tempo_spanmetrics_latency_bucket{}[5m]))"}],"tags":[{"key":"service.name","value":"service"},{"key":"job"}]}},"name":"Tempo","orgId":1,"readOnly":true,"type":"tempo","typeLogoUrl":"public/app/plugins/datasource/tempo/img/tempo_logo.svg","uid":"tempo","url":"http://tempo.service.consul:3200"} \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/grafana_agent/agent.j2 b/ansible/setup/stack_observability/templates/grafana_agent/agent.j2 index b92a20d..bf1040d 100644 --- a/ansible/setup/stack_observability/templates/grafana_agent/agent.j2 +++ b/ansible/setup/stack_observability/templates/grafana_agent/agent.j2 @@ -4,7 +4,7 @@ server: metrics: wal_directory: "/data/wal" global: - scrape_interval: 5s + scrape_interval: 10s remote_write: ++- range service "mimir" ++ - url: http://++.Name++.service.consul:++.Port++/api/v1/push @@ -17,7 +17,10 @@ metrics: metrics_path: '/metrics' static_configs: - targets: - - ingress.{{tls_san}}:8081 +++- range service "traefik" ++ + - ++.Address++:8081 +++- end ++ + # grab all metric endpoints with stadanrd /metrics endpoint - job_name: "integrations/consul_sd" consul_sd_configs: @@ -105,4 +108,38 @@ metrics: # target_label: 'test' metric_relabel_configs: - action: labeldrop - regex: 'exported_.*' \ No newline at end of file + regex: 'exported_.*' + + - job_name: integrations/blackbox # To get metrics about the exporter itself + metrics_path: /metrics + static_configs: + - targets: +++- range $index, $service := service "blackbox-exporter" -++ +++- if eq $index 0 ++ + - ++.Name++.service.consul:++.Port++ +++- end ++ +++- end ++ + + - job_name: integrations/blackbox-checks # To get metrics about the exporter’s targets + metrics_path: /probe + params: + module: [http_integrations] + static_configs: + - targets: + - https://master-01.node.consul:12345/agent/api/v1/instances + - https://worker-01.node.consul:12345/agent/api/v1/instances + - https://worker-02.node.consul:12345/agent/api/v1/instances + - https://worker-03.node.consul:12345/agent/api/v1/instances + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + regex: 'https://(.+)\.node\.consul.*' + replacement: '$1' + - target_label: __address__ +++- range $index, $service := service "blackbox-exporter" -++ +++- if eq $index 0 ++ + replacement: ++.Name++.service.consul:++.Port++ # The blackbox exporter’s real hostname:port +++- end ++ +++- end ++ \ No newline at end of file diff --git a/ansible/setup/stack_observability/templates/mimir/mimir.j2 b/ansible/setup/stack_observability/templates/mimir/mimir.j2 index 6d8d8d0..16bd460 100644 --- a/ansible/setup/stack_observability/templates/mimir/mimir.j2 +++ b/ansible/setup/stack_observability/templates/mimir/mimir.j2 @@ -33,6 +33,7 @@ blocks_storage: querier: # query_ingesters_within: 13h # default 13h #query_store_after: 12h #default 12h + ruler_storage: backend: filesystem filesystem: @@ -68,6 +69,8 @@ alertmanager: grpc_compression: snappy ruler: + # Somehow the consul address does not works here + alertmanager_url: http://localhost:9009/alertmanager query_frontend: grpc_client_config: grpc_compression: snappy diff --git a/ansible/setup/stack_observability/templates/observability.j2 b/ansible/setup/stack_observability/templates/observability.j2 index cc1b7dd..a88f641 100644 --- a/ansible/setup/stack_observability/templates/observability.j2 +++ b/ansible/setup/stack_observability/templates/observability.j2 @@ -677,6 +677,99 @@ jetstream { } } + group "blackbox-group" { + count = 1 + volume "ca_cert" { + type = "host" + source = "ca_cert" + read_only = true + } + volume "cert_consul" { + type = "host" + source = "cert_consul" + read_only = true + } + network { + mode = "bridge" + port "http" { + to = 9115 + } + } + task "blackbox-task" { + driver = "docker" + volume_mount { + volume = "ca_cert" + destination = "/etc/opt/certs/ca" + } + volume_mount { + volume = "cert_consul" + destination = "/etc/opt/certs/client" + } + config { + image = "prom/blackbox-exporter:v{{version_blackbox_exporter}}" + ports = ["http"] + args = ["--config.file","/config/blackbox.yaml"] + volumes = [ + "local/blackbox.yaml:/config/blackbox.yaml", + ] + } + #Default config from https://raw.githubusercontent.com/prometheus/blackbox_exporter/master/blackbox.yml + #Example config https://github.com/prometheus/blackbox_exporter/blob/master/example.yml + # http_integrations module is for validating that the grafana agents scrapping config + # all the listed integrations must exists + template { + right_delimiter = "++" + left_delimiter = "++" + data = </my.crt -noout -text ---- +|Slurp certificate from remote server: +a|[source,shell] +---- +openssl s_client -showcerts -connect gitlab.cloud.private:443 /dev/null | openssl x509 -outform PEM > root_ca.pem +---- + +|Import certificate to java trust store: +a|[source,shell] +---- +${JAVA_HOME}/bin/keytool -import -trustcacerts -alias cc -keystore /var/jenkins_home/security/cacerts -file root_ca.pem -noprompt -storepass changeit +---- + +|List java trust store: +a|[source,shell] +---- +keytool -list -keystore /path/to/cacerts +---- + |extract cert from json a|[source,shell] ---- diff --git a/docs/902_cheet_sheet_docker.adoc b/docs/902_cheet_sheet_docker.adoc index 9ad811e..44e4ae6 100644 --- a/docs/902_cheet_sheet_docker.adoc +++ b/docs/902_cheet_sheet_docker.adoc @@ -9,4 +9,10 @@ a|[source,shell] docker kill $( docker ps \| grep "amd64:3.2" \| awk '{ print $1 }') ---- +|List containers +a|[source,shell] +---- + docker ps | grep "amd64:3.2" | awk '{print $4,$5,$1,$10 }' +---- + |=== diff --git a/docs/902_cheet_sheet_mimir.adoc b/docs/902_cheet_sheet_mimir.adoc index 8967964..ea6ee62 100644 --- a/docs/902_cheet_sheet_mimir.adoc +++ b/docs/902_cheet_sheet_mimir.adoc @@ -10,8 +10,37 @@ curl -H "X-Scope-OrgID:1" http://mimir.service.consul:9009/prometheus/api/v1/lab ---- +|Read all metric metadata +a|[source,shell] +---- +curl -H "X-Scope-OrgID:1" http://mimir.service.consul:9009/prometheus/api/v1/metadata | jq . +---- +|=== +.Mimir PromQL +|=== +|Description |Command +|Getting all values for a specific label +a|[source,shell] +---- +# All job names from scrape_duration_seconds +group by(job) (scrape_duration_seconds) +---- +|Getting all metrics that scrapped by job +a|[source,shell] +---- +({job="integrations/agent"}) +---- + +|Getting all metric names +a|[source,shell] +---- +group by(__name__) ({__name__!=""}) +---- + + +|=== diff --git a/docs/902_cheet_sheet_nats.adoc b/docs/902_cheet_sheet_nats.adoc index b97d68f..18a6880 100644 --- a/docs/902_cheet_sheet_nats.adoc +++ b/docs/902_cheet_sheet_nats.adoc @@ -9,6 +9,14 @@ a|[source,shell] nats consumer next --ack --count=100 EgressStream egress ---- +| Create nats cli context +a|[source,shell] +---- +# The stream and consumer 'egress' must exists +nats context save local --server nats://nats.service.consul:4222 +nats context select local +---- + |=== diff --git a/docs/902_cheet_sheet_ubuntu.adoc b/docs/902_cheet_sheet_ubuntu.adoc index a50951b..5d54c3f 100644 --- a/docs/902_cheet_sheet_ubuntu.adoc +++ b/docs/902_cheet_sheet_ubuntu.adoc @@ -15,6 +15,13 @@ awk -v cmd='openssl x509 -noout -subject' ' /BEGIN/{close(cmd)};{print \| cmd}' < /etc/ssl/certs/ca-certificates.crt ---- + +|Slurp a root certificate from a host +a|[source,shell] +---- +openssl s_client -showcerts -connect gitlab.cloud.private:443 /dev/null \| openssl x509 -outform PEM > root_ca.pem +---- + |Search for apt packages a|[source,shell] ---- @@ -57,6 +64,17 @@ sudo sysctl -w fs.inotify.max_user_instances=2048 ---- [[_205_link_inode,Manage Inode]]https://maestral.app/docs/inotify-limits[Manage Inode] +|Extend systemd unit without touching the deployment +a|[source,shell] +---- +#create a file /etc/systemd/system/d/override.conf +# And then wirte the extended content here +# For example extend docker.service +vim /etc/systemd/system/docker.service.d/override.conf +[Service] +ExecStartPost=/etc/nomad.d/nomad_kill_pause_containers.sh +---- + |=== diff --git a/ansible/setup/stack_observability/templates/mimir/addToDocu.txt b/docs/addToDocu.txt similarity index 100% rename from ansible/setup/stack_observability/templates/mimir/addToDocu.txt rename to docs/addToDocu.txt diff --git a/docs/images/nmoad_links.png b/docs/images/nmoad_links.png new file mode 100644 index 0000000000000000000000000000000000000000..88330ebc122b413d1df63b54c9d136c7dd0ec098 GIT binary patch literal 302029 zcmZU42Q*v%|GpYUTdUftJ!@3a+B=jQwc2W_UAw6nf?7dUQCo}}K}%_C?>%Y;C5Vw2 zwL*-#1YrO-4rF z_0Q)@pT`e}%ab?3PhNwKJ)FRPwqA~8+Fp*1?%r=bz}By5E-$%=)YVjf`Q93LM%}{Q z)lqwa$ucA#!LC--RH{RVAqZ}4_h<-MhvW5Sv{sYAgB5GY&&AW{J+-+9o6B&{@ptlVSK7;{Q}|A7 zEvdG~izMho#EZM?o#s9lWlE_7ru(#*#o*u_%`}9p;My|m$Y!9}T zAj%O?OYHGl@xRA3Fm3s4aYVGfJ3whcYl`0TXjrUPmHy0Tl~mR|;6UeMocLpUNbszh07hc56gDJ)OG;zcQlqxJ{#QDDEs^1*0T? zCi26R=f?i92=37d=FXEf{D>$bOj6j>G0EoE*V*r)DJzPB-k|H8pQk$-J0+P0_%RBY z%b%rhYL$0r)AF7CIaW$0P)%AJbE4=S45ZH zw>I^u%~JcY7n^5fXn2FAsNDP1C#>La31kH@byYs2E8<+-0LJiV(C(QQ{9w5HE-sYg z@X&j0b67psa0JoM0QV!fm37FhMe4H)=L=Lh{*}@scJy-b8AK-3$a4!_UCRziJh$y7 zGc5$3lP^Y60Gonuco8YuY>bwDBh=}`VSjn)8-Qv~JnI4-As?KZasyQ(RJW|jhEEmi z+r6pp$a4;JW`E$N*9fFJ*HaYAIkz9Z7(@G$?HGkB*V`y%VD`q1cV?fw)hk|Z<{7qd zu&{W4Q)878(-l6n|db?KoB1dgku|*0te?_g3 zPM1?xQ!$YIXqGGVaU@@Qymth~hg)cBriU8!$)LgV$m>UkkC#_q^KpaXw#BUXKG6I0 za$P~ty9=b>YLS z*cQ092|g0@t`Z;m3m1B}=rCZV7`jKlDDEy1Nf3XwiV3Pu*Xv{0UKyKIkevHE`<1+A zMI}J1Vy@?iiRgDFRdj80-?QD2U8#5Ld;>rEu(HSLrFcb}?sd*PPrk_dR~qD2jhh!K zyObKk=2M81<_ZX>iJ@AN1pOM9wqq)Gz^`&TPvVs`n88`LyX~7N@i6IMaQBOi?~|_; z`tSV7_V6~(t!0hdb2Awv&zfrz?#!NykZoRJ%|c2dR;3Yjjbc)6=E1EoN4MDlPG6S$ z6-;o?R$awG7iQ0T&p?L>Y$k7QnwEl!GxrO`tQtDflut{Ba*kq$%qvEm#LXR$FzzpP z9kz>)vldA?W``vPSx7qumGe9U`GfcD7*_-+^0XgMnWXmrR0q@J-mkMv^Oe3QA6;;T zE-v`!Q`4F8KH1j?Pz%?Ad<(ZFiY!+GzeNdQSIlIEbUzpMM#jja;#ZM%MRdZ0nsNy7 zIlB_!fhc?(5B}~wlT5m%1jISxhb=_imX;CbdR?L>rU&09UX@j6_Dh(Zji;AZ2DHk& zY~1@*JNay{Wn!wN>etU0{NH95Ty}MNRr1pBpH22DC{(5Mbb5PGf`;(X_yi&d zU07FAy^)dSy0L#?qSChl{Hs-*z3G~JS~W3j<6L{)y0Tels79}>ZPH89Um?vV5hPsE zLJAsu47fi(vx`3nP^e0BmFXhm;DD zp}w3Vha4PZ163|N90f#fF8M4kqC_{s2eY~w2X`Esug-mCIqUNQ!4U=WF=Y-Ky<@vZOTk%-O}hu%khW#ktP8 zK7I9TJ4VoL`nI-VaCpv!_Ha7>DfVk##gIXv4-MS!uDk-xP0wuM`yt zPOW=IIhS{@x-wx@*9al1cd#|AIWVn3%D~`-T zawW;aT(;vAJ>c)t>@1LjWP3$=HFK6dW;I>08zF=MJqJd_DQ~^sM0(^QjbNA@x1;%? z4=zU~SUwxO))TtQ<@nq;XVh_CM_CZKI@uZ%(l z-w9phbk&!*Z^)I@o?A@5uI}Rsev>ihY4sfjUUHl)4k}q$-LE81*RpnxKw4~#%-B{O zYzG#?e0qm``|wg6tTrA}7B;o(apluBCsWy{pK4`W+=u85;!aB|LG?2AxJt8Xo9f;| z1K;^GDSXTIwp9Ws896b(;#MJ=piWOQ>mw1_wxt?h!Q5-}PVTR{i;P&7>+sSxswQIi zuR^(^*LmhECH;){`N9V%XDG{7t%zQmt&C`!BVCM*uOs#gKvOZ{Q@(Z`U#E{6gZd>~ zXzu`Y`3%H)cox>4OkXr86|575b&zKuUa_;`#5)(Bykf06diAdsifOYm5 zZ5H5fG;+Pb6KQsQ3e_c(S_qp_F#bAf25kXvj<2znWuwajUzOzf(9QF2Os{q?H$GRk zWMH(8e60N@@F+qv`Ks5`Vt%hAzmuyBT#nA!ptjVM{phk~<>dLg)4O3UeZpd!9?L@6 z>u^y@8{dVey#PcE&1rv8wtMgio!1lzFoM5epd?+-AO1vJl4Ciq}t9FkLCE zaWKufeai&3E0q22?COn}GJC7@ix*KyxWi|BH>l_}V4z0h1oYk{a`rL1^!u7PR_`jk z>*RPsYK%%=y{NW!DDQ8DGY&2Ej&1tPE7S2y0F~GN>Mu7xNSSgSGj4X<=st^YXkVl` zl`Cx{SeQtyyeiIUnqhPCGuIf>3S1hE@SRvWnt?AmD{RH660De*jICR@xlCf7MKa~4_DUy zA8~Ee5ejTl;Ga7-$~+KI?>Ufg<+8ZO{sL5N-?Z~u+K5t~ zB0C)YO0RHJ{^N-V9+w!c z8}<$*Jv|RGt1OyPIoF9w3#QNhMAagHM}c3I%bCNnY#cH=Ux*5OIY=-~##;(uc>(R+ z0|O`WeAjKgedXI*S~}?yZ7_&pOow>cR{x@jSLL;*+itZ8SO`=! z{J9}Z{^pvdU$m%&r6S!`RK-{48PhWV*>}Y`)o7%jqF?@mf{H<0U`ee{yG(uCyEw5+ zF7i)Tns+Ki%35sh)0w<#?ksCsRcDvRNf|Q0$jUgt^V5wQKk2v>e~UNmJ7vAD9w}65 z?qR#*T3OiBT;jCoTkN}q={a><`ld2jv2@!ncP>U^+4|ia)8TkJwulOH5A;w6>GS{L zXe3uW+K6~@kB2AEStsz~|Kb7|(5#V}6aFOg?>~TgPT4*;&Q$YrY$Jw0TB9UuY)ktR zU)z@&w_Zup{|-=B!(2f@(*Yh)p9Fmnty;AfU*Uo#rxY=7OC-ZkOZrs6kDeux16o8A zBfrER-75PtqyCrS_8JHgnnrPg)!k_s>E~BEp4S>onFc4N3j^G+;w^LKbpBKk@qaXi zmo5SiL$<;7{Blha*~}&jOPVmLElc;)kXOJOG&4~sC(O5I)31tYX|JJ6gq_Rpny}@z zV41yKU8J|eVkBz9DM8}_!;+sxn`puzwl=P!?_;)n)x8+fjrG1o#e{O&1SFgf!9;wa zJsbmr6e-;gBs|t8!vy5fal9`KKAg3{Z#6r=J{UhRv8nBPSA_H0f@sg$_+iVbA zWtbsu74pj!Zc}zUWxwvxGNQ74cDLXaB52s0#iR1%iA9O;u57W@`L98@TM4;DfcO80fQ69XF*4CRA+$6!Yghi4NB+6>TZx6(f9~=B z@W^iux$j^10+yG1ZGWWyHvodh3GPBTkcB*2RrSSMY=uk0m8jRsY?<*v=?T)EbDTkA zFs-G)&+-h~Y+LP<^hdd_6bOxS)s@L28xKE^IiGrXt-d;ha$gI<+$}@(ydhBQl)(;9 ztT=Smx(J*zQW@NB=IML-6cy_l%l|#LKqIjl#0P7GVAE1k(RsWYb3TWxJEOY_S|N>b z3R~~lAHGRMmhQ;o>4CLq;{}4^EPB33!eIivKERtm@OJktuXt@uWR-cBLdT`tU_}j7 z>!{|WGRxxG&)15pWH#d1E=x){3w5g0T(4X2$)`55f1IlWof<|G3>QXriFl6l$wPRys=!gz?QIHT(KeIbCe zk+R}cS@SA{U4{Vei^vc}L%8t>rOlNAtakROM0@+1 z8*lm+yZB-*AK|N!CzC2Ba&AShWIQUGGRjf05f-IQj;V9tBV*rA=izD+0di;C>IJG! z?vwZR&kyP=(qG+B{B5Xo^sf1|Qn1*j;;*?WzV6dR`3y3~*MT-r`JO7{q4?-wD95d_ zeU1m0I{mUCsEX-`CR{%8Husm(4Q55xninr_GBS=(6tsr?({rd}q+92B4G{m^V77%n z%)3AB3OnfSi|cwV@qO-}OaG_))E`H!a5DK)YomY+E_d!9jd%!!ygWvHB)ynf1Istp5?kw2T6Xrv^zUt0AnAq9x^4o$d`j4Jcr=>6<_(Y4NA^> z6E4*Ig4Wm`W~#cP7Jg*gWw?G>nCUtA^2pwGgr^injp5~zK}_EYcTr@xO2; zP-%#cPzjH`QM1^UWO~vjhKU@eFRVPO(%*WBp^p^JykMm1?dsh5V@FZtNs}iNihIETEjU4`+HN+t2 z8;!^u_jD^rxj}YKx6_|#_iF5x-^V$ogZ20dM8L!^XI$Mz#!TtEPyBdqS+mV=coU^f zs=zAWoH?&~O{K+`+OvA6&3@NqrP%XPVb9Onwl3xn_pj!F0K1a zgY0~aaiktMdnRA7ml%2K)eDH9U{=Z%s(21Ao#~mvEA?oKyQzcJ@$l+id9Dn7YGmql z9vqR%M3bXPYI%(OT<85wRaWQE4Lb5sis~Git4fBz1YUo_lZidg-i0<^;`c8>sdHJ-9kMUeB#mBa? zcj5f_a`~~g;Y{|a>EI)onFyA@4reF1sKSKd7;^1dUes(kpQtd0U0>W;b}-%rTH54P zJCt2NSU5EA@e1kpdZJ{`M8Uhr+Yg0Vw{Y7iaWj$fE0z&Qxf<+E%`nUC;XiM-yPrkE zT>T>${hXxKQ0p(Fix#oBZd6n)57$YR2ajyITiR7yzuqRzNR|Oi2wIf?uApf$+fO4*?uF0M^H|5-a}mJ58;Fy-g(3RQ*33B zU{z#L5jxK$BJ2vqRa8eQL$d_K`;Y&FBVRH?gZoUnIdexb^Y`{2D@ z)5mT7&Y;$!q%ENMmC;{Z)G1dntP||7b1KeK*Twcu*1~>HD{E25*S>4d_k!ZUckFnt zq#|n6cZ@70nj(MMKe(jDt_B8KP4z{D$C=DEO>SUaUHCVYN__vi$oj8ZXUW?a8NjLs zf6-*M_GjKvJlnIbi_Fcj&28QnEWYdc0jbLt|HFCqOc-3CV(g?q+-hoQr;}gdXvSA+ zTgp>)jWd;dQg)L6yHwrs+-~UZ_Q!VA!YLV9!!^}D@!A%i({)QzkTBy zh$KP1*5SUE1-$IJaU%rS`IsTC$O)LfAy?JbG!vqH#Xl#K*+{u^CXgp_p4m{I_i}He zN~i2yl>_dHsJ6rUID&(Pn^Lf2kW-Kwme%sT|PI@VDw>_0mmu{hF|e2 z2#?7f{*9`%cO$)Tv(3RP@3@(Az@Sr)?Bin;&Fnaj^Cbj#Mqi$0dx_q^Ybz8>{AT4t z%s)hF#Be;CeN*=tU72fAij`5fS7z4kxK zwu-~vYH}}eCA4UiP~Tv8#zJTDY zQ_wS|CmHz3s`uJ8qNIXCN8i1vV%JVH)7S1yxgGYD^Xmw2_h^3HX#K`qwXFQc>#t-> zkE5hG82?o}>sQ_E;S@S28iG5Bn=n*_gL;(ooS7{wi2;)gBfnjQg+on1k9Ky|7 zlrBvgHD4Ii(RO~O_`!ogXF%f^ZX|uZqw#*xR$|I4^NoTI2G}({J$7(fPPxr4G(fTW zDg^yg`JQNo1IHLrO8pC1Cjoq{CdWzMeXw$S>-5S%p=r7G^T^P#zFhAd+8$GwPT``j4^(i{5Y|LQ@$Em0_-4o@>%_aZR^C8iU%E? zD8B8q7=N9`$|d|$1(8#8?yPQz~y0rb=q2(Mly=hvHev zf%OKNwS!3qRwfD_>8~VEz}oBe7I-Q4QpmZMi*?25eT(#&boUm-r+5{yF{p93uu}f< zy)>weLu1c#u4<~hz0iM`k&-6k7+DfoQ@qaOtlZ%z%0aAC}YEII?nKoMoy z{_uhz)1BqP1MM*=VeQV>7tSB|1ssF(rrHI$AZz5u{jVB^gTZO^pkRxp4-P5Lu+2tnW_5?{IQ^A+bHME zWj(oMY#({hbzAN_Y2-@w*=Rfw^xD=ox>>xsW%jR0BCA7iU_yO0;`rB`H&)*Th_V}s z&#srnRRQAy=MEq)z<{~TIe+hEdY6)U*CB(flApUhUm5_hA8g5C=35hVtyk^Y2Kr}g2UO{u#JD4>N&BS^{K?a7d150NJ8@#Wt2&c6Dldy z;R^#zB%Fbuh7mB3NCfcb>8GZbNNjH=0t9q)k{{Zb5Y{>%aqkAor5J~WE8!-Yu?(&w z!o{wHE^u6o>OhLfG>Ts#u?Ee3dv$gyBIM>6#YMv3S;-d*NzPSmir1+G6=)t7NbM>u#!3mrQ~a`thDw9v6i z7Fb0t3HS_d?<0*8e4CEWZVOHU`f@ZfbuZ(htyDn=sn2e9l^QbkC93W(qg>HP%RZKouzNc>Kkd#y=t#Czwri~^r^G2d7 zoNqHZ>8Q)G+*OMO+r1fE!CdBT!$Rmb8qjGEiR0+L%V;E-yoX4o9)Je^sRh#}AN@XC zseZe`_>3#ri~K}8_AXQEbII5ST>wEX<7R@(dYT>wn!bTL(S8Yzkgy;Uv8XO5P@7@I zy|5dU+44p4?g$n^V_3Z9t~}PXHaOhw0z_JcD;uc49F$mB7LJ*Ar3_7ODLBT7-M` zyS9lf9nyFNyW1G)K=uF~XVUy0`f4a}w5+jt`fASmuPdqAoZJI8g$Ix&&Or2EPuSc+C%1#URpfNwol z5W9J00LqkW%a_wqxr>88v+jSY_Jv-Gij7#DkR}*w}Y(jEIrzt1r0>0QzUDIt-?^ zV^p^oSQHe+L&19K4wPkAj&epp5rf=TZ6*6jHd>|7#q&l6Oe7uqq$*)6+!HN$h8h-@ zbI)wyrUne$^fxFq>Sek`-ORdf7>_#zok|W0Y?(nTSeqTv4c=9AuEDV59^8c2tS86lQjN8j?@8^1-H^H_sHT|IQx6zE7 zm{@#GJ+8hNSZ-#usaougUesykAD6nM3X+)YPv7XdfIg(}p>uf!;q%u~C(!68aDh#S zIe5$y;h}4zv>v;MPdm-6n^tgfolYK_tay|EN@+>er=LEn$u8Z%ybWm^X<*$nbl((L zdm*YwronK) zn~wIw55{=+{(HP*;gCYB7*`Q1hg;kLv@Z1rIQMz-(bt-!$d^^CXraZAdRktg4T&Cw zn#@KgVG`lR;L32qIDxF64A3^#oqJ04mARj~n}0R|A!C~Y*u)~in`}DNb!tCSb9nq4 z$FzyrzSBTzM(b1~NlsV5(X4lVATI#DT=dm@Ysw!Bn=*WV+wjDTs#Dl_vJ*<(f3gb} z_s2PvHc3ddCa1Pzh_05?)4oeTX3uIWuz#cw%4+3hVcfmW+r#q9v1?p?{J1r5gqr4< zi1C(+?XC)&%!a%Y5ktGx7^AD4zr9Nn6sxTI_39|1xZ<3`ux1Z&@k>Qz*Gzl5-okrN zw%BEMi>8@>1bOt@;L@XkwN3moms9YA$_!0AAXi;0Ce2*%!+}MY;YRRelCL5@o0GUQ z3J(-C-mn}KuFwYplvY{>ag`eivM$6+L0ZSpW>5~^GLEg1n0JSo$d7H&oW8`)*96h| z!|y~5{}X4hkMZNeWbz!O%ecZpdZ}360`4fs= z)>ApKRm_1p8vbf^QaaR>Cc9V8A(r#p3Ip(Til#K{3l0m1B2olKMlC0B6?^5Zp{)hi z$;ER~kiAXKgH92VC7!osZ`)BjYPJ;U6F7K^=ovdhaYtVtUcx+MUoV%N<;M-$u5*kx z9!qWY6zZb*S(tzLJB8ekc&G}G*q=A|gX()?t;0N4=e6L46UWSyt^@q2jDz(ql%Tq!F}(dTv0Xw4u>SFnJ(PsR z+}7f7aAYat-pZxu*A2M|1TsZ}L$}NK9qacubBKg7(MA+EtFvAke=ohh-;{7*OPQ5s zj(j$Af;@W^G;MhLN5*%C^T1?kKUrpI)`3o{-hA#4o<9B! z;xf9TmytSpjmKG;$OSgnilc1ls;#R|b2HtYgU=O3@hY?5qub=q;D6ulx1*R&AhH{n z>$u^*%~BuqRTt}8bfLIEy6{2BJw7}%&7GgT@ffX$t5NW)eemPNa{J3(PfGo8hW zAuj=V*~_)*Rc8B+0~R`dZ#A;t9=aM@f?ZJxW@Wj97qr>HoG8}krXDH&6+iBr!u{@Z zdhNCu6JO)u`!P6jV)$c8`@2`w*0mexax;9(REh1ms*82a;6E{rE^-PrbYxKZTqqmZ zSmKWdYk>LhrsHyoHr>W>-%aX+V6VzhZ=mU=)h8=@RvZ$0 z|Hb7rQH%9Z-usEwm_qZm*eLCrd~6@M=hMTUX$$c28p&5FZqt7*lK0|`W>hnV&5nFhR`RQF;k?q&MMO*b`Nt&|QFg(*db_v^KHjPKC`WmB3y$)ox=^#AN&Y8}>&_@}dvk z^!Z{$t`C?NZ~`#)`|C)=_eXF78m+Pn#x#2t_EzUB?|Q|EF$dShVXOX}&?}FMx1MkX z{(2W}5a<*sgP5_uG}SC7{!Ep;JCrMVSJu^XW}*^ux!IhxmAMpsO;qKml(g@@dn!h# z%Z%Es@kxAvk_EqJ-RN)1Yo*|Yhf4U)L448)da@-T=9O3o`W=gem#tgI_h(hC|J6Fd zVTTX9)kbPBowcJCpRH3paav5Unr7${?O0Zd5fXhnIpDk0`e()~OpgHuoxGuHRaI8c zf`*ec7|ODno z7*>qb1OWgfFZVGBze(M6=(v(l{1DLQpd;rmaiF}W7(Lt>)Vc;WBu)5#eTYtqTYpj(gK(bq`uw2$ zZC)H}$XBbqS|NN=e@qFaj(^4b(0rn#)kML${gvWPfX}&MR@0lvSK`yQuddg<-Wzk1 zwI@0SmIS0(z3(w~9f2>G=GD~m;N4zoeE<5OYwxd$PbxhB zsyt5Bus63-MBtFuHz6DjtY2?sFjneNEW%Hay}>%|`)ZkYsT@;g7>wVu-K}(bsi2#a zyi2uj&kSUoXE3&0@6C}GR%YeSGtHmvf2e-AXeVbp-%kyJD5Jl5tMi|tl5gy(-<3pt zgn+;I(Gt1XHd&u zQ{GPDc6_nm%DONj!KN$Jk{5cBD4P>Q>#vy|y*c~$?CNyrZ;b~M+>&?_nb+u#z@ICa zqgoPy$(MjOMNM(%Ub3>B`5aEnM9R6?0M?2YM4(-bbI?msrzXXg4MUCMuA@#amQ|x0 z*wU&9c(6Ayv_dYa(?N z9i;(;kv@0f_`E@X^OLfAFJ_+gZJ<;wmmR-= z7bEe(hz%vs9K=&zA)O~=01$ViC2EL^K=rBB#6^q^f{ixPxW3247EX%(nM$VwjSLaU zhQH6J?;hyoNbeXoSoV9CdggpJHd3i=UDgINinDU3Fm2ZPAOsNplS8WE!6*B1s-RDz zfPNiu)r4g)k5LBo_%6j&)18&RIChp5jdRr?|FN&jZ!Z1CX#hn!)k@b|osKU~JA1K9 zyFuDeF;D-@dsg1va9_VV#|Cz5Hxo?we4REjsF^U-%}Lf3x!c(D$gs_gR!4fw(7dw6 zXiJLf`#*C_>~iO~M8&pOXqu3H!b(6!A-;*V=s9(M!%7KjpLCU1ts`u!CDl!h>?QBE zw2Cw9(Z5*a$uB*~m)g@pL-P($Y`dM0-sDWpZ3&CPZ?(S5Dp1#BU1xJcS|nx^zrRXo zTlC-F9;SZnfAM3s-=XirW!Uk44Ly1wHWbd!%UXIx||-C#h3^J2C= zExX(Zn|x4SIh4+vLJi0Xf*I^}9)?X0z{>1+dAKdF+F6c1*HR#diu+lV7_hD~96veJq4!DXKi;Q1h zU4GEn`zJn)C;DA~hA__p1a{@PGIgk6~c zZeD6S#s6%&H+h1SbqVloo=~pt`L!xFwq=QxJ3tT7vzG+Wq8u*Sxu(eZL9NSn3xeqg z@!HTlf$2I(Mc{F??O4vkpUJ?X{+>ft{a^Yd6cst`jWig{DY7){6&v4`s(g7&b*J4C7>k<`1BZe ztrAdhNS>F@iJv7P0f0pfRNqsw8@sVSBPjxyk%~b<>M}rV7GhBRixJtxO$1~DWw4Q2}Skcv;8aT3Ib`p$2AEJ%tbEoS^C3o}L8| z;$;YY@=fJ0Db_0JY;H0T6>XS~H0@Y}R8~vekBO&Z{k+a;^gfC|OkSBimP^BzH~x|T?dM`5A_;|IN82n$jtS9_61_uD@#Y58 zeC>qzt#C|GAd=J@s~~Q!moss@h?Am~Z;&yta3Gg4*q3gJo^?3cd8No+@qxr|L0nvP zgCaKx42HRMeYza!GgAKPR--D)GQ-GxC|}x8a&pKye$LuJ(XZGPJ*9=>JlSV`E0h2j zx<7DgE^eCys8Yt3sPu_d6V!#KGg#){)T>NmbWZgqF*E`AB{5n5-&eD0-c}p^*Bm3(Y&;E-F{z34M?HtV+;)ylDn_>pIIycuJ}i1D%t= zOY$fdY#=;wQTE+9sH|k|OH$ASX1}-HQM&&69qXFH;;U`ZKPbSbRh2eoKo6xdgPcwI z2V4;d$1?>^CZe)^$rFkCD}BMesuwFtxUiqqC5!0vQV`my*s@#a)G4ds67%MD6FJt6 zbAn}mZJABCDt^f{yB(fPl?43u_QeM?yZv!)e-m+*Z}{`S@xB9mqOV`2`G7TTpE-y? zQ9u!rlwZ)!Jr%iKzL)d32?RZdG+pYfvBVl$aN!`eEu;JJSwrZ<{(D_PI$BijUj92VuDh3-zuX^1}Ge*r^O;G3& zZ(elIk_`Si6aXojxb*vVPDN#b{Z3gm?;=$y1Bz5?oe~VZewbGezB6^J+(^yBZHggK z6MpmS`~DG^!yo@HWWgBKHE@+4LhT z#g%Kx)F}X+lT#q^{QjkfVBw`})}bFhjIn23a*Vd|H__e3pp zETY^#TXm=ii`Z)2Aix(0EtO{6h_o$Joau|P!IPt9Deu^{{BVK*9_4Dg4<0?~Mu;A`=FEic$ZRrv3+Iy5x#aH1Ip z)Ltkm`Rl~Y3Cyht$gfqt$d1WhNS5U2@*laVY^_%$c~S+%+pF#$?JUrte!@^NDW+xlthKChIX^34mhpm!?7i3 zX}c~z*Xsaj@YpsYg37m(R;U5%o+rv3RjEi}H;1lSl70s5kZ#t)bit9I23H!ehZeNZYCY*#+LwE-{q?u%=%^>lYCvwoA1EZH2ZtKKW zm*wHfXGpH=X<~`ujS)Rori3ac-n~Ix$md81=JwX$=`Ltj!t}RuL>^ zq7j$W(iwCtaOXw;TTgwSO4g<6M>)BIhdcT|{*dPELO$g(Jtz1z{)@?5+7++zn;)MFFkJq{`aR9Vx>?PyeuPCT-gnU*Y1b>qU z$~vg~^CMowJQfvobkf%L+R~_7yn|tBB&8{I4hJ71ho>LbL!Jc^#V!HVKxlfKp z7<6;qFEKS;vNOjyLcBG$T5kE$E4ovpB#7@&8H{Lk9RUel=C{PA%aphKovC|8ri?|Y z*TFyTTeX3CH9982+ z`z+aL;MgmhpMsV@emnOg36#_$*rOh~VB;;Gcom}CI<`)s0BF~p5#SeYPd27jljIEZ z3{~FU-8(p)`miK$236zsy3dXN;kkZv)|eKQUFWKJ`9=$v3{rsIOwR7~DaF#XqPLSp z>BT8iCI4s&Ob88|W!d?!Pn8|6+2=PM2X4V1ej6$}T+3GbByI2YM@Cq*)bp@gsmfX1 z@;lRD5085bvUzxUG(g9-E{GDxU#>O?RjWgK*ym+5Z*a-c|kmJ3{)A7z{X_-KWXN z$>$jRRx;K6qajBBnK?P~Pm)ox10a4${cvO(L75mdE%PQNaA)Cn(qJuaKS`fDt0K~y zG=`xhB_;GbHkFVZ7>svWS+VBbR$dKHSrVNGa?xajy(InX$JMM&;WmZ+57BY4p>Fz6 zAqK`b{R2*x>i$mtU6%tNGr7F$FD%t};oGdv*-Yqr?6(BfUOX}6ofpb}IE7;7%a2}48NE=2$h)}#8_up6_^2BprBRVlN3`uuR3H6T=;ntH z@4Kx}H0>N2i1znYbE}X(<~hL+4`9kJ>F;rIK7KVq;{03aH-7H@^I@bs4Gs;rBpLTi_ec>ZpwE?Y>t*4m|m) z8HdXP>hc0isx_V+DT zy{f9RA%JN{SotmlZZV{!nV_PW^1lU!#<3Q$jM2Zwa=96@lolswFL<9Y)FJ+QR;x%m zCmBF&vka`TG=Z&BsCf`}mv^8xz&xB1{!gr%;5Gx{x#SYhQnrc#g;c5!7d3 z_&l0UWsgJYw6OVdzE(kYxv@eo=-*b0nono5@sT`10WEtGbmu5!1o9G=k?j3i-rTt!(flxd`fq%}m2D`cI z2nv4arQ-SKaPwL}At|@B(#A&b+3Q(erlPRo_^X{}MSt7pI}U3%lPRkDj+_t4{Nqx>u%Tfg(+bLCe1wIJaQmvFSO zwSSo)G{^hoM{;1I>9q^zAX`V&s>t8xaw^@r8ptR zX3~^1EJZVlAl96&Rl)6E)Q=fMcsC1@CUu#hKU{-`E5k`QwISDxC>&A{36JKoUDlOv z2_wk;i;y3^zh|ZNJ)wUynD|vg5q_+*kw-EM6i?2L-tx)5Q^rm0nP8+!bkN>M{}J1q zXRW~Esu8|NZeN;v|GsZ+G-9;xUf(g3oBXt?FjSZzihjUTimu(k{9HEowS-ahb03@4 z$5YC(w|8hIaJeuGX5aL|zzZE3;oRxYcC!s2JYlr8A8hF%k?v{G)3+sUVb@p!gYRW1 z6t3F?JaB#@7q)7yP*d9CqVG>jl-&uAB2`pd1jO0#+C0@OZ&f^SE0~&hop#O3f7=wb| ze&iI`d)yFGD!F8NLfTPgXcTYhUf){Q@ll7&(tfiVs_rhgH_J~L-PX5jtz!VFv_b#Y zbF=gG=J)YMK2DQ&8Vr?G+2LZOSM0TQ_HN_l_V{yW@QqS_Hz$%OmVm=myPFAb@A|a- zRXx|Y@HM&Ozxmcic3znQ910GiA7Rz#=47N-7i1_98QYpE^R|+*%Q?Gc<67OBOb9i+ zCYy#_^|^nYyDjLY)=KNN_Re*f0dXrs7)%UnarXM0Jt(&zV>fR?_8&ccg_=B>$BPkK z@P1kzt1E}kZV2@T{Fy5ECIcn4Y%?T-l78ouMFp`1WsPBcN-zC+M7sn(W10Y=&An&0 z+%aKW=vbD#j5oVr@zjsX)$p&Jy04cwE8s@Dr)xB3dRVt9R`K#-Sqd1b7R%UEhQ1}nz0fx?nUr@s)S(ZTd38DH$hI_ zrl8=e4CTHE&kraDl}i1fEt|g3k;k8C_YS(ND#gqKmoBc1wC;2Tb?+|0c6rZti`K{R zCK+VI{ngtngL;11)I{jiG13nmcMRqlWH$AW?uuUCJYpKGy282g+LiU_#x84vemwt9 zdCD67)&B!%q$2~-XV=bY==bbj;|NSN z%90h&5U7BypT^$u#uw!`e3nRmZid5hEUgXR3sc=c<)3OR!!%+t=*u2*^NpX^5^&gh z(KmpTV8GJJu1_3fh;m^zJ-o#EW`~p3C^a3&+aAL<6w@9b)^y<+4S@sX!M~Y!q|;`^ z%`SCFDGH{Hb5T8b@_*<$^KdBtx9=wv5|T=G8lkUH_I(l=vQ@s3byAjO-^M;usmQ*B zvQ45?c4HrAEXkIUeK#0P42HoBW`^hLcR$DT+|O~`&wt}T$9(3x&hvA=-|yG?kTIbE zIU&;BellVR{i&qQTz6$L=C{jR3DwWMdR>uzJ8x?NUvpiz>vl}B|Id@tLn$CgsO3_( zQsHO{?oUI)?mt{cGDE+NLEHpgS7Z3Ou8Lyqp{39Ihc_OhBeEZ+D*8)Wl2H9MZK>k% zOKj44>bMkYNCs!E8)5F{`)b0o%)NTP{DM+{RA^6)Uep@{OFXl`k(c{Cn+!LIZb}=I z-_jk(#7*1NM7ysBIp^942?!~Myysd25ASb5=TNWj9csF&{-67VxcD1E28J`mB$4p~ zS0;hTE!D{+vD&ap`9&Tn+XTXHj}Hw>a8u3(cielwaL4hW0%9FKqEwQ;4cOF`)K8mV zBCx}i69ltEGsNR2R`9yK&dzFR8~a-A2VWL%kM$^JH{58Rr9o;xJ`Xe~At?XmdEuf| z7m}sn_nX8wcx^LEIO-Y)wH>uoQoTPcCcGZ~Z=N8;x8WcVqWV#@+8DHNy zCP0*WD@nN=drKJhF$d|^q5*GwGxswTVv$3vH;tI-THxpFe;?sGL=e2CfZgCNTAx~; zO5e$B;$htW9ci=Qi8}eX%`|JmeW8P3nBFI4 zvLyR2t!V$9z(vQuKc!xcq3GPSA6r%qUeHKumDEM`TJG_R7PeAjRF z2i=zsO=!}*SU48#9Fc6jBb#WyVw#6~D_pRWEE|LF3Zplqz5SA=bvYxG&?Ak-a+O?q z;_^W+18Mo}ipFcJ4R*<)jJ#-dyVVT|MvlI(c_9_ zXE|YYsOz^Wfrw!l3Te1FVbpWFOW*M=q4#R3v*sTY(4MhO-p)41HN^J@1(43=JAdB% zTG4a|!(-#(92a)pRe(BoBF$82OIJv&p)msPzB3G}abd$~O%EXP#iOo53 z`lq;J*)8%|l5R|=7$&->%O@-0n*;u6X` z0+ZxHxj-^yTk4h`KA;iPT;3UHJnbyQr6i5vP*yr#t7!JJvz<=s`$ld$we#?xu*f&)PCx1>BCrv=)SJ1;%)QV>&SrsNUg(ri*Zgx zQ-@$pk*$Avt_ELv)NayW$ySbhBA4V{QH}_0bnqPvQY{R8m%jYY)-Q2rDaIBp5Bbiy z%~@)3o80xfr9c*+qCVYK2D{v)LPg~v`)E5kkHNlMmvWPgFpPz&!E><>bve^tEG`bV zwgIDt^Bfl>`XD&obzaH=34spR?6ngx)rC!7p{(v&cJ^5s z0{mdh<=>tD2RY6~4zNr!&zv`=_Q#OUe_SKa3*VXkKV7ATs=*oY)0Zn+q!Wy7#Jcug z=^MqRNfqs-KG1fcOJ3su!|(s3S-)2RT`t{}lq7a;JN&kpG?parLB?JN8bi+wF7>X6 z?jH`yhT<+r6d?O1KMfd->U5HPQG%Z#4*F*U3HQzpSh+zr*Yftz-$S2@XnY1nvHv&`i;5vvi4)} zkhLQJIon1}b5dq@%~c$#E!nJ}uWazgE`c{XXye!1Bgx$Gcv4xw?s^sI@@Ap3nwoAz3uw0 ziLcK9_uk)1h0LOe!j3IHtrT8JLZNJ)(aB`&^(>EW*=g*>_dU3capU{(@(r5O zIRp2dm&p2l%)Krp6%n*%+U@d|n$b{Dw25d??!`>)r|S)3WqTNISE)x9wxW_mdcL&i zZUW?Oq{AKPAw@qU@H}^nJ~39IIKa7*Ep*YB<(1$k?r(|gH6UM8YjUS_+Z%4wCq_s> zL+bQt*=~Wy1t#*fba}gHrNLOIzB=r)A<=O!E}o`G7SozjWk+pMu(pq=Qf>YP$JHJ6 z!MrH*ojSAhB+X;~B}u*HJwHN8VZG#K>e<$(CuVqm?oCQ>ZmIR}7e3TB%U1joQXW{Q zZK?aDGcf@cn`xR89O$2z=qVU4g^ZwF9dMf|wlUyruIlS)^x*m6e@z*hV^W#B6+9e4=KqG@+7-Sv0CB@ z%;S#*CK9;B<%#Oey3O=Szk;N?SPCY^(^0fvwpMQxfF>>H_vRAQr#3l|HL~8} z!{4pfss=#bGlx340y2s8W?09dYv$!*}rtXf9d~bM4SS zXai{O6kko!ApAxkZ1=ZaxbSDtoCW+d6pk_f7??~*v%kF7B}p)@W5|FgeKk9GtcW)$ zg?enS#iY(cl65st6hdR1MU<{`@{j2!M$6EgIaqM_Ehw7#w6q_J8P0Q+-i<#@))-&fBD zv#akK5WZZ|{?~{`KIaZQ`8O;nY-bdX zWjk>qFsb?|EeW>(m@4jO-zWY^5d*;~d=a5|qz60O;)OAqMU+^btj1?Dk`>3SP&See z&0PjQQ|1(S#m`tLb1#k1*f8=hd;_H*_ah|LC|(!-iLvsB-a{fzKFv%XugP3|P$5%j)R(U7(G-w5g0E?nnyDK1 zK5VmuwQ@jf^FMcLeS)suotuk6z@-G;`vb(i3}=mzeLV((`<}w+1eCd^BWj~(D8#`$6h!LQ}XCJTuE$Svs zv(c{WCUMmF=&Sw%u0(n!1a%v}mWXar5Q^q|QyfyMvt@{=;tZv1bZH?ypCm=a-h&mw#V2hb@qKuuF{+W3)W;@R%x!GU}gy8F(n4;%fc35ooD=T}}Jj z*V2sW;42a~hJG)^z+AEIGS%1uz`F1TxNt-~b+%Wm2kaiQ!ZwDn%Bgf_j5^9rGnARIE*~h>L%E7Dcx;izV}X2;5Wi8#naZ>`?98>6-uV) z>s}~vn)TF%x<${by?9fdte7t8VjA&fOa+ms*s~E+qrc;|tJ;=_92$F*j=*g&2hVw6 zXhNeAon)I{5xn1CZF}9P0qU|A?IEdo%6_j`b;W~oJtGcT!J&_Td#ub-Eurd_*5tMwYsJx~B3o&bAD$BEQ3Ws8(yvcmGI)%-2{P=gI;s}1D+|ve*L*h#=8z}AZA?QOf6Ls36EL{W&+p1qM3cOY}lBG zDs!WGwCCS_G_>|xom{1me8jlnuI5O}?&&9;`wK7cLncLRq-J8O)A05}zkOnIBY*Cz z2C}1~Ij^O4o0F-=pLIz;n>BZbEeq&5*D|XWm-9L-mjr&r?_S~X7*!DM&|P{r^ywSH zxGgW9FKymlNNvMeYrVR^etQkIFtvnaJ$xdKjPTzP-aph%3gbSpKoJvFbbc!2)QK(7 zsWM*7Ax|Rxe%fpxZgyI%NjH?tOlV)9iAH0}SxlcSt|Fqpj8&Ee6|Y zAWlv<1gw`I9h8GEml52+bQ>)vYhg#z_=*4q4RKV*JQ%~)A{d>BY7_Z}!YK1O@U-?N z_`!_CH&XBmW%Pek!bG{kW;_s#XoO52bw}gpqNiix54QYLEVJ**zfv>491&@Zm_$}z zk+^f&Bo#ROqyg83Ikt$%SOWTwYv8@b;jtzC70IQc=Q%gEDJI%&h0<5c19t8k_7s2g zUEtg!3>r&hFP2~b2)Ju<5Io8E!cr~mk4dBhO6*t?XU`pz(!$> zTHp-WE@f}aBjEyiN$iyH%SYi7~?uG8CIkZ~+u3ha8+wiecL%O7l^f7u%txC76SeLXs)gM|LQANEu+^w&QtP+W|Y3Rb|*MA&8pl*-`=S1t( zH$;DUj66xrOPlIE;L`1uZ(8%9wiMc^22o9{G!Uo<3KJ_`2j1a5yt{9EVUt6>wU7eR zNN?TbyU%u|Jq<(F*tx(fihDy}kFMY4AFN3TbKZD$G4C4}|3P3si+5IiqBLzWVq1)1S^t zwNNxhw%7On;`E|Lkeef_;C+9Z+T{VraIycF==GVno%f&88{fcw6HMX#|K4I z`k&})-eikITQ32WA{;$-X4e;`@+-94Hgm1w`#vDpw{jOpz>;(-9~J8~IA@#LZgd?` zfO;MnrBXKSS=zr|RSrx8;>wS{4-s6JAHMWX7X1geA`%W9R#2$vn}v|)>hUs4Blcm>#d!8=FIx!i8H+j$CV=Mz zfy)wq*n%u~a(rO7{8~|I>GckMN56NX2665yg`iNeE*V{UifOuZ|0}v`AxrrCe=PC_ zULfs{glCS0{*61G`kHfM|HjMSKNzw_6O~-3_41UHLOkrPLIn%+)YNai_H(at2dKAX zaaZ4t7V>my%^F^SK7#5J+_S?-wjRw0Yvl#qb#KE_ystCR%=H*)6*9|wY|^&N8v=^W zngdF8!nT#D=?c2fI(c;ZQ(&}fOYbzsQY?%>E*$n*@q4LmFhw!M;0fz8(2R;piX3=J zM5D}a{ube9LfnSZ*rE+(OM`Ok7I(fvWdKe?;UG|$qFb3THel&m87C%>G|0X_Kdh<| z$aj&?#Vb}CeN5{I(!b~GKtL?lH2(xC>bm`xW~B#tyHxwms=|VML4!{J>PSsTJWgDp zZ^>``iCuSlr#&^1Z057s;zXT5;ZU9*j>S>+X2keX#4)1KCFzcL_+4R_wAM4y# ztU6g>)<4dV2}7m>l||$8z;nOOGh(uvJ{Q%t22B)dYCOg!Ef5Dour@Be=s=~%@>nb# zjYX}luKu~QK-%Ka+Dc}Ql*3)dpoqwD&G_1aBcH=%pOXCiR-Hg;gF|iYR6Xdq(^}`$ z-&dzq{bxW7Q5c*h0G}Wi8T_|}#Xsc4F^(N!SC4{I`1dXAozL?Nyr?ZTR6p4*O<X$~9fmdm#ZAE;{G(U}?${uQ39OScx zBED-o4a$vKh=n_+k;};yK$b|0Qgn6e_Q@+&_IDSJ!oGOji7*}K6v>x`DXtwjrVMpD zeckPkuN!*f=u+os@0I55=q?Avdk>MPo|px%1^!?@@MwtgsYMKBc^}n|%p80lnW^l3 zMxOLc^ZDCzQk2m|HHXvUFd#7t7YKLuMxnl2Ry^$1* zs_WvG)t|1=^BXZ?@579Em$3gv`Rc~HTdG(Z-S!o1k zk6fExXQzu2z3Z}0W|zXZ3_WaWx?M{qo?5yo)RM@N!*3F_+9_c?!Hr>d=2HLku9<0B zeavp_`!wcq$k?Y6#E#T2PYGuoFXs*7hP{E-JWJ1oAFQm0EkrBm(m1p%iMPjn>gV6U zK1}&wnFqJ zCFBn456hxAK<{b{-iYhTs6Raxs#rE?q!0g@{#pM#_%b?KtB8NB!vlY1)&Z5+te*9c zSeiR5Lc#f8?V=w&{-;_+Nn%Ye#4QVo4+L{;=hv~K|)m1o@k=O=Fdar)i1S~ zt2+rJw)hc#H_49w^J6c@vxAIEekxeQIGJ+t*az{P+N%XGje)!XMzx0T-f>+GUWW2* z-Jw7e^QyD9h8efpB64+|u?h-IjJepIH1!{y{bL`blGfkxu6yIss&Y)qZ)(~7n9j5m z=uYxwR6q^xJMGbvwYLW3LIK~&=_;;?s@ep0!gbH*)o#Km+dvb>?vMid-5duni(&_? z?p^QVrKQF+DV^?D-e1VUSfwZp8qy9+BDaRM*77}EZD+hZO1Cl+RM!)mRhc-S@#-n8 zS7{wHuV(f*#zjsI>ush+u3Sdf+AfvKu9fxvv(6x3omu04+OibhNU9lHl$Ju?`1a%H zINWfSjcb9=IoXtP`!R~vroh${mlE9bz~#5V`RJoF6buw_`yZY0)8rg$7$gcD7g9$xt;}}-3+VZ!T`1!lQ51OE`cDveE}p%pXFF0NOHLFP#O z-qjPbb(1?C9_#bQHL!sT&?*-FH|Risd+2g{S}Q;Bqn6uHqYQW=UPShH-{YAcT&@;q z;o5Pof#taai=!Qj_1SQ;7c`D#FYQLp2H0Tk5xbViaj}R^J;~mmFzxGuOQiGXkBsIX z4Vw3y`p>AR*(f}TO*rPR6-zX0SmOIPS|OWr)Y_sWg;MjkqckghY<*Djbyo+pEIA}cC_Pb9S^_xt(!|ErF{*f;t2ZkaQOd7lH z<^AbYUAxnBKuv-4Sguo*n~#r;7u2j8&tOWP!+Xw$mKlRe?EwT0@#699fez3bjvE3u z0jfdIRg~TZ>G2NYtS5htUo+P=F`w5Ok)N*al)S(#zH9V=R0SN^;2DJVS8!yVP=B_w z1r+@&i|qSqW7fK$AJbP);PIR+#n-HGD|xpwYip)!$GSZ$SD^T(7Wj&mGVC0$kkTKN z)w@pGuLi_)+M1NlI~H#zr+d{I5iQ4bT{4;Sng#>?NsA3`C0w>p0t2FSlnJ6kS$)?N)zn;LejefSq>Q}MK*?_t&Ks;gX0F(wSn1w?cqxKKE3Hg)3(V7yqjcnE$N<+^>3K4|I&f zP9V>7z(cCmSaq8>k#85b^F74EG9+13`@bet@3|kk8x5@&aYjri%h(^WvJ1@RY!>32 zHDgVLSI$YwhT1bEn$CJ8c*^Ru0n+JCP*CZ%b3O38l+NFIv9J%5Efd#Z5%4n1B;(?Q zxBlt7>gVkv^lw<~49p?dECK*vhj}&U!rC5KW%Sb;O-jV;e(^>a&cxX_o#uCY&vjM3 zPgm{014iuNvI?5oR%K$E2Z*y@W)8zu>1E`J>M3~td^UHv<{)NhUn@0a>Qr$9NhAEv z=m*&7)iE3VEJ0*T_)m=MH)Yiy5u(SL7=21P0a;C}L ziwq(TlBFH9owBy(x8jB@XUEN~*R|pw(S~emS!y!u#FL3)L995q8K88LYDz zTJTa$=&zQuaSPVf9x!M8ny|V%JGLU}<;F)7$Mf!IY8#tgsy=(RvO+CHv z@P+6xc6zPX)a8XWF_dfHv&^MagDCbFN5$BKVi`s>n310gfF|y#^ROvu$BVtLK(RQ8 zul=Vx6jS@#9j6 zZ!4Td9w$`v+HQ<*bTO=mbpy}H@NTJDblV2{$rp!!-2zxoH?*2ED3LN-Sc)|Xge>x0 zIAjk$V-#uHMM&hB4*Ag}jve-EsfsD=bPDy#rm_$-tzARo3qAB3at^I-%$?WTerdN> zOLv`o_l3&}>`??Fp-W&?^J0I4W>*|{Fs6N#xL);TA6+{Cp>&k7=31OfZ<^UP$juEM zSh8BE|KDVd^144QdX=KRg_Uuckn`m-i%}Rp}-oS2#Y5bMLhnLs7H=X~tv}kdh`%9vJhGCuI&Y&USE-M7Y zbS`aBiH2KP#b};K;lqLsSNg;m%H~Lx?b5|vdY5urW~G@ z?G{-+n^xmsFWLM`JL&hvmY>$lH%l&sFCQ-{Yg$1QnuJks)Poq2SA zKY})F$O|C`-S6#J1!&3(EmtPy@=`-~G-kJQE%FnDH59XY%=4K3OZH>+m6(@LSGDZp z6isx&#%F_n#PALT!f*o;2)&hKfp_wV>^6+Z&}1ffX%dKJhFxl}auZYb zY=v-K!su zNvEy@$&m4s)6^eZmEp9F=g(Oy-1o_)8hoWzHpYJeuubvvVeX6dc(V7n?KGSc|$;ZGqS zgROe=Rny}FJ7hc-8_OU*2V!3u;ekEs16Qa~*Xx?m)6D6bPGBR( z_o13G#~DK4+Wwd3#$7vV={n*uVyhHUg*{3;s>U)4kby1ux_KYW5g%O)e-undfe)ZGl@3KPS?TRyhEis^tly?|k^`%KX=v}PGyzjuirgRApi;E$O2qzuJgi@WyDA)?l0_U?Us zX4JVTo^hgXu->r1&B(LCMmKRE?j>+WpMOh4`85}4M&aU{;?rDcM@HP2`-XQg+){$Vn$ME-681T;M1_TC}QDH2l{;va$sq*q-}zk+lEXe%>}VHvT(2I$DcXQBg?| z@^!u$YHe)|DZ{alVP!VdAW~`FB(U^JAmQaV`A5U8` z`f<(GZ{^?mzm-qOWR(A8x91dt?>QyScXIAI-=f)i>D&zejin+_9yQ$2wX?_4H_%j^ z7I_@Fu!=O;7j?rCz{ymqjeed53f8eCWo%hCe857 z(p%vB8`0F_V1)b8?rixwc6L932|T--q`F`5{q0L)nh~3vY((0j9k!AOzn>l@tHjSQ zLdMcC?dT2RT(VZ~hCl=jyJk&h)G;X{MW7c@4Mt-h^K26_kdwUKQ3BOyLe?nV(-nii zzpc3TkcGCE8$4ShcDRXO1LpikKC)IN)g}tfP3uBC5noELXHn(bMEOEj-_;++tez)k zX7U(EuPIf{0-#6rP5^JO8$Orv+G?z+FUzgjJ4ne9)fiKP-Pp_mZO}{LRBp%u&bDBbt`@h(q4q@Qsec-uaIoo*yEk3`gp+8)nQW5}H^^%WK$}?Lm`8@w z&?&%Ndm#VYE+aq5sx5EScRq_@j?~qeln}FH@=I?*js7+W?nW536VNxb0ztic7BPk4 zmLDUblssRqdoIIK11?{9zi-^t*)nF?cY0P>rqKB3(^s9%G$cdvS@QQ1gXG5JiaICP zqn}&orqXowbM~sLYf}IGGtEQdU<3{&U+jd~=73%+hc~w6XQbbphB6lTsVF4#+N~_P z?&^r^*;Y3#g7@|n;GzvahYYiDx|`307C3xrmcEas9`?`H@xTvF*sP-u+ZudcLMQ0F zaCfps6MZyVRM$5IKO0KYaRY^nu}nFHHU^yC*ZOct%pjzb^^mMi5hG;s&cdgGHge)+dX4A0|QK(M0H4&Pk8cK`_vXq?++=gp^A`GUb_lSF$p5Ftxs6Q(kqlkW}v+rVn43@f^u58 zkol;wRGk^Tit-pDN zv#AN0lEZTv6;)^U)bsuM8FZHXxtw4y+nA|k(?%od@?(?8En|uG#SW}gQeaEKSJSUY zGMUx>rfmvCWgn|XLtCWzo7hIIC{rfxRX67yx+4HK`Xj;rdXOdOigK&JDG8VPOJWd5 zgPxm1cWx;abm<{lK#q8q|o<;G@hIP&chxE6j&8ZF$ z9Ox7C3uxS&Dk4p!aj|JeBCM9bJ!9B$*E>$D%G5lgExh^lr` zAmT`3p@nV3K%QgsfE1Vd6$Bz!lN4gXm_AKo=xVYU52d+425w_i zZXsW{iBo+VX_rO#+U60~qQ=+T4mZ%7;^`e|Y8(=!yjoWqKBoIu^tJZ6-st%B^+^}p z$WCyGJy_@lLmN%$fp{(yE{%>?4P?1_qQ1Z3Qmug(fL5nUYJ-YKYS*e8-O(jqg3u4@ zJYmiNt*W}Dwmf2;MM!O4SXRc>Sa3Iu6^+qq%lo^fW@zfpsk}#S4wMJ)!!L&~AV^^wVYSi^+!L0up2F zS{~I`NPIAM^#UHj7e-p*3tfJa9B9?tp`E+jhSf=uPHdM+IjAX}Yz4_9`$JAipz_WY zv?0o0(Rv5Cu&o^KM@dI}Nngv_Ttu|d0+v3)9)%7;c^@MxzO5md6Uf{Bh3t)a_f#~8 z1oPZoq$OR_{(Q4sb^c{72VnY(DQUYl@j{r~(ha<}z{fc-tXP>z91u)@=BVP(*J!Ih@R#%!`D* zPEWt}H9!trUbnPF4Io|^#rA)LwKmjE=+nSYM$#3gb_Bvp@_}U`nlER8DLS)SB(X$) zyiW6a7G}bh=ou^tFP(w;B;qvxuMbMg#2Lau!T z4qVvDtywcVMiayn(dGqA*K0i|6zv)V-K){!mc)U4URW1>cMOUzi{phICPP@AI$fS< zP|A5vHruBvr2WUiVWG=-Q!whiV9|Wc-OWJ@**)Oqmm1VNa2i=+dBnQvfLZ%=&|gkh zw{E{)`a-7R`SbRZM_B2??#RIuM!Cqf!NAk*6bC)pM^yMczF4`iV@2bVgxiPIFQ0;c zym-|Ny7Vn+S8}~YOyrOGYX3opOAEVZFUvy84y)dK9tJm46c(MhuMrWmn?}3lj-WY= zZU-E`h}>)SS%(}^Ine|vn?v(`U{};5lEaNzbMTTTnz<}hTnwbPYoAomx`I4*E42VvB%8QQUJ2-LfD4&>&sF1O?+zGD0+Sl9g};C_Cf< zLh`E_yVk1~BZ~5R0(SZSYgd*(WRh;XAJad+KL(!>^>%cv zF{ztaJeln^_^DLYWraI?ft_BsWcRoY^SD5#IWTvmG-4ua3=kcO<<&xBS)NbKn}46a zlN4|OEPhhHkGGN#H32plNf-T)=AeTQ3E;`AMLFkQ7xT89NAHrxAoE%0kit$_QW^Ra z_{O}hQ~B3Nq@zStxc%?1T>uT?(@%a7b4&kV>xlQSQ6=IP4L11{`#OW0pbA@j1J7RCn#Bs2~dglWL}XdDkbOWGa{?{knlaTG;IDpeKjg+qQ2kp=@lkKgSpjskTb1k zB^;S0fA5AUC!o~eg2#`jK1Zq)8V9{w8ntyZqe98-E@(}-k5?tkKAlSQ7kMwt4ClV`z&VhE7re$fhdv=$uj1*Ygt9u2T5UcS{`4-rbO zMcmhO{D!RN9_k0vVy|=SI(PhyprK@8u1zmNHE;WWX@j9<5eG)H*=21!7o$J* zCWWgnaJpBy8^v)pp5|*DB1F?pM9X>uOA)SV zn&s1BB4m650HFn_CtK-4&JAni&OdSvf;#qeCAN`&^YFX35?@N1p zoq7~BGUZ9F#s@evxgev!8p2bM;cS>qBtfd!{h<<0@s??hv% znn*sTeAe2_(zW@i3INdGMWIWcb$b^(o`%^jRpkP>7pv?l z=K{2&PfeU-_2?CaiFczKu=xkpHpbx5VXeGOAcPv~d438nF?)S9UvB;hdoC|5vP#Z) zd1*EnsAxcxLc0$@s3{v*CEyM)RZndk#a@RW;0!Vics^c7P?!Q6H_ z@6WsVq(ZDkAUaB{KMBtV#zPM@qb9jtu3nqoNRE%gB*J`i&BqC5`*6a}d*(VG;}hVD zeKulQciXwS(i_<>UtBsP=%uRWHEQLNb=aiCFeu2~Zs;wcwMb;7jajkX9>R+l_gAzx z=>ur&FiI$_LoR8uU^lK5)(Y%2z>Gb+NUojyOb1}wPb~R;>b|?!c_|jG_LtTDe30$6 z9FSe&J$fI#@iW}BB{{_$`pawEo|%m#?<^~YXZN@6RxPCHbC4eLigIpp&@<%hfT?y; z#n{*wU|~+EI6@w6BQ(##1f@+YJ-obX+>nu@=0G1y-JAQtXHlJzagiR4*o%&l9J(Ri zhO6EI7{++^JNmXcWi74&c*e3?fuSJJpOo+7nuk{#JLeODCfV0HnTjxd*>f(n3w>AQ ziP^l?>=SVE`S!Tw?HYub7DEwqVJ=;6Ja5fs^lJkKs)Glv(~f1VX&?{ z_+8iv!C){CO~#;`TTl6sH}==%8ISyA8X5J`AWX7U_M7VZkHy#XwRGSi7QXchUnJzr zt&N8KuJ0+Q1|?f%M8B+7ex)2L1}u_MtM3dvN15H1+56DmKV^@HqKDE>5i!`o-TB_V zWbLIF!nM@%-Szf_evQK_keQM$(1t9hOAwH@ypux&aNx?J zwZZt&(ovS29mS!zZ2GaB8NKwIQ&EkoPq4jD@rY`Za}t;t??4}6^W1q`q~R~BAQ5Lc zEU~h0Y7yL|fJLVB=yc$k7tF)Iy5_5BqbF7`YP4aUGe@pX3h+`_I}%G206b=#8hSg( z3ccpyVbk3GeOV5rB^Eh}W32ICQ#dHGuZ8#a*9RKOc7C1$2(P}~nd>h`HQTCgb)GgD zr|+!rMGnU6CnzNX{DQ;^`YVSamH~~3z0E)Nevm|AcL0l>!~~{VPQ#ZLZn6=zp9_~b zfzP+)Ng-b?z9XY>nzDiuM3xb;Wh3!@>HqirctKj^O)@w}ne3SGOgjJGT;8$Jgo_2) zG`X8#_j9v07U<{kT?%Dl(j1?oh3D*@a*7L%X{cPPFgwubl$LC ziw|xed zMeG4kSrkOv4ieIo=v&xqox0_ul4q-7kkPt{ol!5K`5`9?5(nK z9SciQkBNf|J{fG^D)ot!70jg(r1MYRbu)?s?J2IB?zQn_9vzX7nAkro56IST>9;Ab zM@0q$*?}&4-nSDV*QzT5uK9zHw?^rA$umiWvrb1covl9#2BYsv^H4G}z`Wblpp_)Y z6@J}kd5u~|Ld;<<<-B_9L!Hlg;$9i4T$iqkf4BI`3dGZA9hw6RQ>u%qsuTsy?&mSw zXazU>gZ};}h26og+eM&8X=eJi_Q_IH7r5&Zt2fK$I)T4)%96i;=7wJ5@;I8G7#Q3( z4=oRnTbShAdnMAgKTEgC4GWHvDJjhx$v-giH+~!s_e{-MJqDHH=qh7ZkY72+uU1tj z$2DTJ2P(SmJi>jqS*DA7@d4X82yw6YnpV_c3JK&6Ry<1uKz@&gG*nj(eWcx}Vn

$w$XAFu)9NkprkD&>{znn$qO z#BfH9l?uV7_yb`KP)?`5^RHD*zMd)xoq&1RU;xJ?#@RvzS7B0y9ypaK zOp4K%R>Gg=j4N%C$h^8)n0~cpz9`o#b2bvIQ~9*vL37iLRO!@X6@^mRyyKOSYYWo> z{WT{!FIK1dbHiF6gjlen!^k#5+ybsU^U$MRD9DCeRKA7vD{F255tlG%ZJXQbbK**h z6uCW<&k4(*^nyO&^uO~`Z;k_Hmzy6bySiVmeTxY#qBu(80Ou{Xh5*Q&PU!IKdjm;u z=f5iWxn@Kt@e`7k6Udu)!bwwl_CNl*sjr(`mg<`Ir*ek2x{F$e$LAuy?{tIMxHuW< zYJvVAk+kbu17=TNu%ZzRf*7Q6s}sHx@X;h$*EulnRnKQ7#-cRX-2xf@(ePYC=Aur! zu5(5B&=<4OIv+guZqI<}c-marm*LPw*|XHBp|rol64>kZQjlDe?4BP`)r){@2EYS( zNsDcvi}~KgR#=Swbpx-TY=c|#Xm38(*_(?>nGz8V4Eb|7eg)T_r0q$Y35HhxdJxnR zr+|6NqqB3jke;A68#B1{arXK3ZFtd8`e(iImUvL0F9!JTs0W_p+2(<<(oMJ8>x!v- z6&zq)yXKIsg|E8vdjz28e;Po__Q7ZfURnaTZRL_9I`cvnIM7=^g+OHnB1X9{y$V-Y zwKk=p!ozRZPQEcS)@r;}b!Mwz%~#qi%Nen2jkvH<#78ruvi)&>mFuv84Yk+0*IZX z3+!Pi{WQ=YE>4&&Q%S4T8w1+lq&>_9tqo<6t28`nr-lXM)>gC0RlEfAO2&Q7<6i@| zuccX^Z%Bbd#$^rTQK&M^fM{N2i=gnm^2v%`DW6Fni!%H?Fh%PMDq&B$eX|jmM_;z9 z^L7}1cn3b3KrRc|Jmx2d!+3GphKUW9}<5GV7r#SAd+Yjng8?W z;AfKp-?p8Rvw}uzPA{*8IfgE!%k(c|%gW?arM(32{P`GhiDj7+) zFMJ%@^MKNAo%uqg;DYQHtS)+cL|vK&dT${P1lBBj;MM`+npoL92DayRP1+@kS zq@D<&lmAl1;!W#-fPn9&Zh+rV=o8L0aj7@c&X}VBqcK1%y~@Vjg;rgBQ*DAl>Hqj& zbiH*{)A8Fs3?d9f23SZ8P(V-=P@2IIQ7{M*Dak3_Il5ygU6Lal(%mWD&4|(Mh|!FW z-^cHLKfn8ao^zgmIULT|yRPec70S0J{CB%Y0`i9WM1!YidWvgn zHE?yFpRZNj$5VS?$mlv9uljj`Iy8OkxxizVwT51?Inm1>n_@}Io*~1wrQD>7PQ3B6Yv4RC;wo*PU z_97#%e=VT*I>Q88)ITkGzFou(+c-JgoTBsJw;u9+vm^^bKRYlUno5tL@S9ac0GjKLFmz8BYhn+As7qkx{IJ&1SnpuYDc6NqG& z{jX2;-&G5+(~U$%j_0l0{^uPdr9EK3|Avgo^XkZxW%tIc^i$-@MHgO63q|Qi(*FU{ z;bjEy;v!$sZRroCIp*sYieDZt4P{K$MbYtS-9KrPy`SCw`*wVU>Ed%Nd9lpB*nWf2 zzWu*_LCp)`s+Wi^nda7K$M%B!BwvDhqC73NoC#N&U!=0b?s-*^~prsdXPajUCzKt}T zkz-JhJZ_@?mDuImvI}hfflWUY=zeTK40)~RU?;qWR9U+09kt0jPX@5jw+<`u$MWP* ziFfA@vW!Rgmf9B8ULr{@d!GL1WucZm2S#CRzGYWv>e-+izwD$p_Rg}Oy7qrlc@G5^ zc-`FoM*j2+nQC-?ynFcXKu37UpdYkzCRx8)R`lmxb-`ncl{gfpa|>#^|6E;c%kJKL z_GQG9Zrft^sLE8GmTYLW^C(HwZgRu#WFIH%5k3!~#!vUycwQ%KXwGLOTK;0u$j#K#7%sR~UG<$gbt+7`)sN0J-GMrQDL|^7UP%0q4)b?~<0uc4& zev?33KRI+q7JK5CO%C1dA|X(o+$M*9k@^->cfHwzb$Agd?O^}Gke1kh{z*j324h_W zWW&E(ooOboC5LXi&(_x=osh2z0Mwqo&-!b0lF5o*+mE|LT;*EiCkcXP=D%xoHkVTz z9B%WsA1JVaLEF|DP5d)Vd?djFfO@B^^EXd(TZ#yc%fY@s_i}vvqdht8*CGT~1WjlN zmYXp?xj-yTL-!$mhZ#>AJi`Bmh1s4kswRt~=u{Sd{tV(8_cP3*ggVw_VRMK_W!HOrn?B@2)Gp#{SKo{DJuOSzJc{z$vN}98uG-x z(LMf}j!sP-+M(r3TI5i^5@`TmeU5*d2@z$CP=3aY>xM*|1jjqB}Fz8_Ml&j_fXny}^SmmdU zZmXIU_H(?``k!5nE;M_YL=WG(Id?XD^pJizek>z`l3e7HDV=oT&As%t-?YCT&ys+^ z<|#a2;&xstmm|zP4LtKf>P|erkqnwN{1pS+Ugi?fOu7`BM4}FzZD!nA3}2fpH`^Sz z#1H%u^B38a)R&Q2H^s3y9n=pTC{9yheRFPyNfmr{U_XfdzHOnq*7bi3$Ju z`I>}4Bv)>--1j3{Y}lLT{{0Mb(&kxO6Py3)%>Nl0L;!^wQ$dx9i28|_7rRrX7dss1 z-tQV1VL02xLeRT#$jH~GrvJ^OQlCTe3*T^d{cvy{?^VP}V5gREz+tfV4Cp9YyPs30 z(ORGvi!8R;Y{Mezw{|#X7DO*`SLZ3TT^%gV`Qf7{o&+WY6rd#b0Rg5@Y}Uw zyRqQl+re9p*JIo&a@aG6U7;dOj%6F6~37l~1{e#Z3u+%_~)9`x*b@D*~!r znGU}!)wvImyY@|Q0s$@M0Vj+Ww)-FyrP?{zC@oG~WFS>Ya%sN}_ui$41Q_vQisvGD`(P8Nw3mKD94Ifrp=x;5|fA4QJiJM*cF$mPsY z3KuH`Sw?M8k@WD&Kp+Jugf67$=sJyYrw}Mz!-1-_%DBk-C3cjyceT2~vAc;WIGR77 zM&coUCZs5eWegZGplU*Fo9_a53oW^;*zj`H+aLgk0S27^+Bu~3P(*NIYg0Gq+|ov1 zmaSeh(oY-G6aHqGY_@WQts8;w7Mcv#MkO_}iN#)T`M@jee z&A&*HC%s?&_ahVdHwipelE68>s*L18u8WHWy#dmU$G*7RcLjtn3Ri^XC@b(uhQXbFb6IT0hIphRh30-brqcd8GYxsH++w_&cuF;x1ewHR3_`>7!my}q(Fon~Hy zI+);{bXQKyGl^$gH`6KZ76cM{)?+R9f)dS3*tVi{CM*>Trpy)V%er!OX1|7n772^) zd06#m)y~)Ora}LlbTGI*oKkk%4blx3y7fy|4nsQ{w*=S!>_;0>=(fVWNzG}pWBt&v zXbl2BK8enMh?jdeO8JHt+TbI`B=UI%c`HTaA3;@e_|g+a>>L;wC~!AN25a)s_=#u; zNrpH7wSHqhp1T!VLaE-y81z7s0#43ImU=l+Ngw>gl!5Q_Wct9V)f1Poiri~B6zpqj zAQz8oFyFOi8GE@8(j(KJ`cHt*Z(4U zX@B}jZT{8y1DxAo)TtksT#Sv28cYtwOuj?Xx^^Rd&j*WpKg)EJd8U>$-|MVL%ors6 zX3&)2z%$Vx3?I@^C)$`8|Be|Q3KVGG!j)&Pn$-0#!%S<4pMUmXIl7<}L0FefE8}U- z1@SO=X8!_YKuV(hEN(*GoE$Xt*X*9{#LbxfZFk|?ObQQ|%;!{Eele$SKlI0Gw0$|4 zC^eGx4U7b^=DlD(euLytJA9g@3|#>^p99Yt$po@T?@`TaY6efMYQ`TcY*8Aj?o9Wt z5~@q46#uC(RJS2YjR4|F%z0HLn$?jNMS*5ZtRMoh?Z{1WYG-tPqjPR*WB!>R;TbYg zI)v)>(EzUtyko;}reDabasDBAON(<{0^K_fF~p%;EH}goUFq$?w@+n$vFPa;>n1BsuZb$ZgA%dmTDG(Ttm$?Dpyp z)|w}G1t>7$TcwDV%`go@Go5+=>5OXS`rX_w<+C}uSo0V&$eQQYUDv%6!r@2|qxoJ< zh-xJ~NU))-iP-`mrOxdaQN>$ltSrkKFw*T52l$$NJf{TDuUff>DvQ-8P;^ zfn@d|$q;%1!(;h2TJ|fOMn!2qQ~;MJ`{l^N`3djO0v$1v1zG;c3cxPIP5kZI)ya-Q z_MHd5rXL4c0n}zTc9I?BnQ<~yHT2o$=l&pnKaA<~c~!!v&KD{D-Bm6grr2a+; z=QL_mW>VRz3oD>WUn$~R5!MSf&PM26M!K$G6yLP)0kBXl82a<&1Nx59rL zZphwT>|6FgYb-uD94&is$7Cf}RC` zP;eR}|D`f|O$Uz?(#t_5Hm@#w5)mCU$^dtgrKHP&JI^D4ts7Gz z^SU1X*KAAx=O8<5a|9g@ofArY?pfRCWQdj!bSTh95b}lW51e|w!}?@q*2rB8paUwR z%bF8e6o^BBg`-`*pSSCe8Y@Cns^rfC<0x}SefL^|S@VPuOr(YQ%4(()g;<5+&c4v^A!^r8@4n zTT5Ok0eHDv2`A>uqQwPR(lZj{79$d91`lmNhI?{8+++c0ofoke$D9o%2HXgZr80*xao0o@K_Fz09-43S z7lu^JH#zh`#$^p?RgD`yn>110h@B!-id!y8+NG`BExFT|%YVYI#k?^Zlvj1d1zn*W z$A%~hPNf?#=Iyo393C*p_KusEchey>R`zR|H0HJd*ma~F@FBUESZq-!ShrI$qoQJz zQhoesL&A^VH+T|SAOE$7X~XXVEyQ>3B!+AH=zB}j^K(7)?RW>VB-I26j-zg8xhwpi zZ&VrpT?b@8{o(Y3ydeGFD*5gP!NcEed!m#xa+?C4xtNJdvHmmKKPf0=`)|k zq_l%<=xyLF%o$_S!_I1UgS)Bao52Eu;R||qblJ9}sVz%F;UJgppG)Xl2$@gU;QvzE zR+&Z|v}qAy-^;TFbMHn?9yowo}ZIQBq6;rzFB~w>9d3< z15&8Zy1FcD^VLwqmy>)ycxaxwLuZ? z8AD6Mq6HG4SG~p#TX+nf1%bXMQ9JYE8+iX;b zVI!WB=Dnt$n#+4hCXDZJI;A=inONHo$jRMI&ml{Xs(#a0^RiRALy;GM#*;m4!sNAT zF&QGC^JgqZ2dejD67yX89wnA;3MN)~ASJ}zlTrJ>3)P3b(iG?hK6tN)?y+nNT)kt( zn_SLF?Yhw&hoITrPS~jC0t(2XZ-sI&;cgEgeQ!%80XEhS+vXLnC3Zy{a~GfqK|uOB zwSALuFER`5UY3l`15cr;+e^3X)rqwv#oob>skEs_Z2C-$0Ds3Tj(;VooosElVBFr3 zfkNPm07r5FQOH9E;b4Sx`u84tcd$%F`8RPFbsSP#z$xyF;+H(h4unu2`azT9>^sWX zc4m-Olz+q;C;_bAbW;As*(TM$lFS3Rr*+&_Olo2;8Rj=Q%B+6B9XR zG9gnGXI59WADXvrJ3wF&_1knr`l<-tp-m|j-&&z`P zqdyahT4SmWJCIxxtT|QwS-!#8#+bNWnBai*YxmP00762IRu+_SX7*O_>Np1x=sR!1 z47py8K%{W4o54Bz@fE*jh1{9}1i3E^tNe`Hygk8vZ1p2yS|_p0Fjk`6MOL6&=VB4a z;WLdI6)CGLnlP*@0yLGprDp1x9+5qs4;(~K`YO+-AIf4;mSau0d(2b7VSaV}{to9} zAyhgcnT>QV;6G>q44i&(<@Q!#J0bZNBBaRikomVw5ul%Vn@oGiT~PIKPZ3zg-qqB- zF)|_8D&c6w{^h5r`Xevx#S57aMxZqeD;y1JGth6y^(|3e3av9;hlap5enN4VOab^?y>19UEx}oya0HDkwowjSXL?xj?^uN1hQ5*i{rM zk6SB9d_eM{+D4R*Zx<-p{PY~64*|>63aTVaJ8+f^OZ~~Bd`l^jo^~3xV)Ah5w3QJ= z__6+(911OE^W9tz)@x^6XWk=dW z7g*Xuy0ME0m2s=$43k1A9(ezdeIAC1@`(f=30UzcPs6DYqr>@{s`|acxq&Y3BJseQ zK-Y}Qm{9QvnCm9$Q19ZEnR-dv`cHgOz-Zw_G~k1@7E^1!o=~rPaSqFM_lQ*~oL&t) zmq{!}l#cSK8w$@J1p)zUqMriL~{|YMN z48-Eb7difiulEVY_R0DaG4eRFfjvH%<#&lL!E3we@~&PrC=yTLDt|9duyFK4_0!TK zy1y`2!NT~xA#HW&#EALZp)7qxdoNyO<0V6sF+AMs?=P`QM1_Vb{8@r z=p>fdhK&|XSP^m&>OJf7LPx(X5ChRJjGkM#{PZKQtlM6+XIcO)aLBoU4ezWPr>lPw zT88^u5-=BWthhuDrJ>k*JZ#VO_*f_l{(GQi7YX(Q0y5`6MjT%4KM<`qb8A*0ouP+k zrZt`g1>6X^C#ZKFMzN&$Zz4m09J0gdoHI(s55!XJOA?=L^b=EYBfc2Pi&EV}m}S}! zt&L4W5s7qgqi@WMMY$4Y+tEK6K6Xmg(9^TJGst!NUYs4t?$t%@b_l^Zx&yDHDDy}@ zizW3S#31Lih>%WY)#VQ#Tl}*LG%BIMh4JrEJ$6bJE9CS;xp*H*T)L=nDgnf-$8g=^ zuWt^v{-#>4Nc1f@x|r=NH(yv)?b9n=r6#BGe0^4}?^a|UT;t;__Bzq#thDl6v04dd zTn71CpyVW!iVhYzuQ>sjQ3|dXuQ;oPP-u^RVHAdEcf|AI-n%RwpCYpW1~$B#4cV3N z1AK5%!^vMHaEcAD4wvR8f8b_|CNl^^u9qH(Fr(U;tHGPljnZ-T6pwFA?=szqG{yv+ z=GVJ@E)je5v6E`s$>w3gPm53667y}^cycxW^yI{DmUPFsT6|*ENU5>SRCzWBFCBBo z{4I1HO*VIF=Q-`pA2SlzIDUXT`MD7u8-<%@lEp67RORK3^dI-q3z;6kYM)ZoVX#CYzO<&ah(DUM^zd;hD~T9~Pew+K_aJf<Qf8H>Zd_2Hk*@glSzi3HTMT(=HT3#FIUlFG7B#m zYaR-l45xR79n;u>7jI?Qt$%GJE%7C+tsfAC6MT!zX2clte{`0b2cO2b%+1H3mN^O) zEJFXGE}O3DMWRWP%{HDmFk4jsl2fW2OvuwKswKWf`=zpqrA^BZ{}M$rB2Bdb9OOMI(x4y zL+jlxpD8cCbtyJrqUY`XR_}5M-6ADD5%QRJ0=x~P4PM~ISq28JJMZ;o(#pEBwOwI= zDVZdN@)s~C9MnZO1{f6`0=|(<9{=|dr9mtfr!iiSMq%g$=~$PKpG#<)5na$CQi?D{ zO44Mn-nWqRy%-22t8E3d!0ImCDc(Jz8dhn!HhA9~a4&LS>5fq%@gk#}D{50c;P>&? zPFEy(526BR{Mi?!&mhZ@Y7>*R#~2TX?ky!eWVCXuqX-CxCks<(j;BXlT^z~kJrP6i$^FKi z;T>lj)J7Ka=*?DBbsku(L%p7+hGs}747tw!K7Ni4?g#DlHudtYe=?V@3>5X#)=vVP ztwX`X58Vy(o|lH25LVOew8SEm>N^6(_y8&a68=REX4~#NjAuiNbCsiJjuV2yvp~*U5d~h~|B{}2Rb2CI))2Y6B{QZd&06!!QRpw7N^_*jvx%=%p?Sg*z|fYiUQbw3 z)r3v)ByZ&o=J6{WNo!rB?fwr4d|Xg@K4_-2F|&OM=lyus|0H;apQ@q$_EKXv-~6f+ zP|_QHkCpweXPU(1N|GfKQ~T&0)`V}z@|UodA8Hm6$V0`1_=w!OA;*$l$&~dVdYU5{ zHR%ve7DBh#N-2fJzB>Hg5xN_0wK!|tLN)YO@KS)mm}VDjaK}5u?0!rpyy-xEEFs8i z9|rt{u<8Z*8~^34js_dKM!<4uK$!3Ob1}t))mTYo(1aJrb9lH+GSJq@526WtqvpTr zxP+xPmWfYQ4h3$+8-7oIi7f@J4GK|G07^tmjPiCQ-HrkMP<$tkl7#s>{n+;BMbRk*P{XDkr;{3u{$UE1Z4V&p#)#_rvYXd z{gHxQ#=z4MT0`SWUYk*?oM{|Sk6u3ws9m=N|3sUtLmeKw_UWahz(9LHmRO!pPZB?+ zlA%5V+&a!YFY(XLcGRvcBu~4h5FHYV>XtT=JzcyGHB2@<)|P2(YtF$rMdk zS@ZE3O$%jQUg;y*PBSzu;aO;~72@?w#2(;6XFDMzJ4SCYr$-197ICbf;diJ(*-SX7 z4yFv0WpDWecWjKJQSlg2#4r{Y>#)VS*Xy1=XsKh)#>e%&9}LS=wq zPs@LLiG^4Hb6vY^I13MvI~`6Kv0eE}u<9m!A}>A!21+jt1rxwS2Bu6PH1>;{W1rx7 znK036c?T~=Ai{|1DFaPcyN_h~+yD!TY!8@8uAt+u2{@H)g-W}uW$jQvU!S9yK$a!(SQIYU)PDd70!@9H_VXZrxa%+GAv6AAV@H)BQraQp*x5Yy|e!jq} z>dbl1@w*XOl28ODEi_;`A}@DM|0uIH7xUUdYR{ebH5=aQk43}zsMRLuYP$omMY@|z zA{zY$#@Hh&T-mElU6IC_WTZBC8Z94RK9&J19WtfcF6$fl3co0Armn(0`vhGD(%o&zQqai0p5NXM4fAn#kQK|1Zg6xY+c`^AJ!G@_wantuKRCn=8I`KkpU1%KbKk7VT^cH!^f4eBC zJjm2Z3R|9yG9)`b+Qz)bMh)(BvtQf+dTQ22&^CztSK#Aq?R+@zm*6jQja z_U5)$o=r|WJy+0R!Ib@JBnXL24U^xSAxb|CN5PDTj+d*Zy z;}UYrKZ7D5^58}ikPt9Aj0!OLYfT^{juC%N?+J58(-JY z><@PoK*iubeKzg4&q3R+qYZ6sZJTV{Or3A3TjS2HKT2lrGyKX6)gQ~u4Nc!-pJN({ z$Vq(A3&crAxLBymW)kt&iD1i7REIl8Lf256Oe3BnsN>vOvCjY})74>Mcy?2%{ZSSsj?=_*(;9;s(k}V?Ax$2#;vjv=vaZ#{nAJ+)py8fnG*8dRDpq zkp4b(*X^H2EDZMF)fmG$(837C@6Kr=Z#;op{B6nN$F%Pv^>d0F1Isd+_9uk$e@|6f z>AHUND_<5f);d-`AF0yAX`d6BO~vqf*8FN9hRj*SHe@u}8B0LT{u1KpyT6AJc8m^u z%Lz=u0xw&C2>rIcj0;&KMw; z8y?M^4rFWYhfZ_iFL`#&Kg)WEVE5nnO>q9DarOi`jg|*@26ZZ5WfvGkYdi3E{ux7MoJHk2DKy_>$WnI1QFH&7 zgpk=9EU`i_NUH#lyHbfwRzu@) z&$um$MR|IO_D~!V=;&3}!M0=5L7C#Pp{-yl`m53N5?Sqi7q+adS|;nP-cFaBW%Zy7mz- z&0&Aqe-_ebHolH6OZPspkwAi20Ikg?*+6Y9WZs6o?hFV>s z=<#&oY!3jO0$t6|Fr~_!XGd$8Oxf1tPy?W!EsZ}VeONd%rU82cxgOWfdojXN$0oY! zb&ohxXS5SqdZUv?Z1t_$3F{>Wr&Y<|if8F5_?8hf-wVfAUfRBKQW?|6IqktmmxAFWenh z@j8xA6to(7it)D;IjBTBsE5~m`DTr(1Zd)16x?R5m{#>)`yR2fEz`(rWwey`r6=f zZo{7iCZADdF|yq^=(J5%IMYuk{IZ`KPy$ZQbfb3f>Q>0lG((FTF8X;lXN~}tL*|u5 z&sd#h41xE{M$*jK;^1j7vspsrRKmS~L?}RgJ)N&AXy-fH@6wj>j|;UwHkt3JSoWMV z*}|WLgy|jfBXkjkds``8z&5uV=;VF(UbN_*&Yg(;V74`S+zV12J$6`B%h9-Yow2fL zMw>Kt>4iR3ZNmq*_7l-767Ci+H0X56#v`2q`WAKoDCf|U^21kvbMyaN<0lAO_9u!@ zX7#_ZIcNE7;ZyfiIFb-WmRgLXM1lgWyXJq|n`&vSi}>rt28QkrHStt1z>P>yWhBrp zIuP5P_xpN?KSq7^e`tds3CxAkR4L0+qDo{$^r>dV^ekPzBKA+LJ^tN7o${Rk>ap?It%>Xqlc$y0@ zY;)h?8qY4^7}#{^i2Ao`66a28rFcrZUwMjkyIUACd!pukZv~mX__2bjNVqDKn~$%- zw}kF~JnzXjI?#zXd+rw8w0`}|D4Unb&Z$q(;yNqrq+R!f^zuOiyGZ0|l`1_t(KQM| zJ!k8Lp*^geJ&CD5`!f;dVOnw8CRvhA0E*+kI%Wy=;{fBbJZEhR!0ODN4Q!PL0LFJc z=kj9$Mkomt!SVj;9n%d7z37$YSPe^$>=lD!L=D@rbEr9BU)Rv;a>Q)R!Qr0d`A%Rm z+2*wl+d7TQgMkRb`Sh&~<>!s>gVZe>ONidOevLh@kjFKXpFJnCWSSR$hdtRp4vAV# zJ9x6);=L@LLe`Zr5os>Yv4kL%)cJ>|ApbLf-t{Cq{*GNNp=C<{nZ0zdjOo-9JJ!7z z)qS%dV2`~MiZYGAmt`$euYp$h9i)ILe?u*C6wI_Vz-2EjEq%sQ&cLCUPzR%w{w90K z*IW={%}NmX90wi{kg!N<;e;1OtvUOBqeQQo-^vWQ7eY7uO(0oug$5k=EARs|kqZ$D zB*1k4nb9PycDV$@gPs!u9tDA<$NO?9qtCuWwqQZLHSwvN9v#m~8;3kTvp3EnwIpL` zQTQVOLda}qqKYWrRkwJT*@HQUhYup|0nTId^9xsnPKDMVS)M;jTw8O4knSYowEM|2 znL10o8U5(5^_f3=DY2(!F20`Qtt&PRUdXrIjUT)00DS6|v}QNzQZ&&0ckEw^9~7wt z&3dm_8Fr18nP7fZ)h#{Pj)a?v6DSB^Lcv&CY2gjqU~FmK&0jrI?(emSCnp)!S~;vn z!j>r{M)>`P(&RP(Y`I$W5y5`$7j8gZd|K4&@l( z_tj<$S4cX(C@`;d1Ug1Q9XntCS(0Cl8`k+UIn^E}IYUOZ2qlD@%M3o4rI&4-8q0;2 zgi4~<>wC^7{Gt@fFWt#RPvX)`m9&#{gZP2h-Bm=D9E#X6qKzB}<=#HF`JdV=^h&TX zEPxcW{Xm|@s9O|`JvfyPC9Kgen&LEPzg=rhXr}B;Rk(G*0Izk?(ZU)hvrt22mScX%nfm~@7#c`8fj6Ymb@3aq-N zN=dmb4(btOLsH0vL8WA67(Qo%+hP#YIG}morGsapUt+FI)ZzRK*k?WkXPIWrB=CW5 z7R&VCb|Mi!Gs4OM3RQ{>Q@1A@eXN2i7%-r4#Jca0QXfh$=xIc)zp&UM%d$$=+}C2? z=RN`>t$#RSD0-nNN8xSf(YtOje%9;vXGc4G+EQHJG7c(PNoB+^r+@K_t$D@Yi`{?i z@Pr1AYfNd;P3hO+v;S;8xCYF-$Er)8QbJmlJF4w%<=xS~q3#wH>*KB^zUOR(W-Ls8 zb}!Ef5X3?K<>^L*y5osu{oaH~OH1@kVQ^k}GW%SKhJlhA5 zhjR;|O%2)ElnHaik!Qu+RxX_~D7S)@J5^#r370zhCvWQW8@&nZlM8@h-@(zZ<`)}n zfzA}0)Z=5;%i_1?jlw9P=uKKCVxt(cJ$ed{3(KO#8C;PYvBI)?t4Vdq zKa>nMAR{hxa?;aq-qYETi9Fz&EDT^=>4C-;m7N{Edx)Rr+ zO~r_er6{kmIs~BcCYEJ3Im+PXN?w-$&(5ry6Qocai{$X528M_9v&H7K?NZuJoi)v@ zBbRBX$dDEGC^CD~_u>vW8H*nWgNa9>!I5UvbmUcU`H;hlgQBG~WyvTF!-sfmHrOTu zqikiDr7-Nxye=ULK>#*snXN0zIqTNAmtakIXUK3q-z2el4U$>xy_mpONeN$gCIIE&Dqkm)VtTI~8WP^BHHQ86a zc9e^>1TCQ$?$Td!)pzNlD+?uz1jHpDF2dAnnWG#&ex|W{MR&F$>xYU!EViL{+#Dr% zA8xdrkgDo@HuVo4#1)^pdEZ@rb5yp}M=eq_co&XaIAr9!r2K0w-O6-L!S`n@m8VQI zwl6-JAJMOmTHTtdWs40sK5DqS;7ZaF5#0N(SO-*r;Q#OXAMa+}Ihotj{hNC0w{UWI z;^Sq3&qT`2@sDKhV@L`C3%=D0(s;w4G|JNxn@JGe_yq8Ye!_#@RU$9^(txX9`?a+q zdI;v`I}kmfST@ur*^^=Oj&Mfgwvhq#e#y@ZKuQ-P!&c;Nm3q5hVY_NFA5t=K_WdY{ zL9A}$tu(X@<3t-6R)4&&?X(Ez+$F+YJAJ)sV*OIAIE?nrW}qVS5&w1Lk>_7i$Y#rz zV|-tH^Ggwbp3n@9f5AHKS!p|W4L;BuOT8-k*;3+7LZ~9(y)Xqhv4NPJ(`OP3Ql09Q zi9nckLBSao38QDiDc{fvUt7=qshQ5>i5*sA)MS>KuJx7ea(sxDAuyvCj`z17fjzu_ zYm9I(J;aYBmR-cZYy0eIPaAt0p1q0h?*H^uW?=LkYcp6~>^BFzmnPN2;8YBCbHtfx zq>#q-;3M)!obu5SMbj=0Fsm907DM+ zGES6r#;KCm>G&N7pRYstkQD99pAhy z0XL1i9VW@RjFMC+pZC_KGC6fN&IWVa+y0p!9!7v?zqh7&dr@`5 zsSk6|5i-6R*Y{w*;9fJ!{_Xd9b`2zP%T(8bDd}Y@sTcLAomVEqTB^H|W>PEuu;EWG zOu;X+(lQ^1Kas4zT4QI`WpbO8@k6!8lS73#=JIPEZN9`(yA=So1`Jo~{(CdZ>1)U7 ztj`)y51x56M#xUquif|&%Z_)u@I4BjMVSmfP!iq z<)1D=(jU5r%y6Qg8?5I`qJykw7qIMK!BT(S6)&#A)wCC;sOZO;pV;5q*S1@Wyos8P zd|+Y!#Jyh>i&ufRE3=rzIvo}a2{^G}U(s7DfXE$KKr0(oFSr|@vy*_l9p?w+%9Cjh zb?XQf7rir>E4MT4vo_ZTgx^d##aIJeoq3bCC>Ywm|D3i%hWyM19SVe$o>ug9SgL}Xb>{mi zlQeZco3nwagUDSU{Yy1weS!@2zh!N|@b>m$I#`FVN`Poh3Oyb)`)Vx0)J0ygiN(G* z?m1#qMeWU;j{fKGv5~0xe56jhh zx@6vBCO`8YaCnhdYCVs1Ii$MlYC?*F+*&aR3bS@c}BR!}<}^*I&A&-HTF zo;_t}52yLU*Oqqu+I8~l(<<7_vy6It8L=;4@m7o`8fb2jI{L1Kn+qkHcRhOW^ty0$ z`0~`ue&4n;AvP509Hjrw!(gDg;7V}6H%Vs;u_Ze2cJ=+-w==Yvxs3F%!Lu9FRi!Pk zgl1-va^G3}eM)1i{J5#s_l;3|NzI`;lmwhQB5_J8`O-?lhP5oINn9M+B=*CNs=G?) z#<4Bf+<&>{tsE{(IuU7;76xL3aMUi0YuWX+TP?_5YpEqMB?Wa3-JZ0$e_&UYWQ7&h zg*Tw5Yc4}&;8w<_7?!K!{iz;K*ip#dPf3(^^1%TvHjRE#!lI}Y_FD;?36_6`VfdJn zDfIqy#JdDms+7bi7q4R|mnevR>?n4=Lu@mX8m!2e0JBDdOvEcOFbm_!`S>#~JSMR& z!+CsK(QKXH`O_8E8Q7P{f97>avkw13LZkWPWm+~bSA!x$QSloQISn236iau{&%;W# zVb!%VolH8`=-UIE-UWLsPghN?!isG?&vq{}<|d8DjRZ#420>&(-C}R2 z?QbNP?o^IT5VutgevQ-;>uG<0TGL3$b_ zarIZYLDU+D5k?S5?Gw)==p0SR?f0aOHms)Rb^?TN!UQ{w$h3|K+4k#=r2J6P(+Id= zS($`Kd2<0CpKecqTKc^}eP{AUZN+bQk1+ol-S$_04o42Uf3=gX6p{ri1ZFB;&V&O0 zxL7cq_KC#@@xhgCF5@H2VXL}>NdpVzJ30ndAdTSpPk<1UBBB&&G?1*FOvp2$O5R`J zJE8NB_a1fat`^Z+bWTje6*{$X3;pF_*5RC|1VQWv8wF=gb{Fj=3Y*@b9uO_Y2LvqL zeSSZxyyXh7ntK}-7?$xgb4<3nz}bsraYRMq4_)OHk76A&5Z2U~F9}K=R{B-MpAKb@ zUhuYIwK3(Z>(Q;XxB*n|nv&IvR&W@YF;{NV3 z%9e#i+l@uWCDy>Vto_rXxJBXq1LN+L#vt;L^29$J*O*sQ+QL}7RhXsNtuvS{$~MIA z)$FbAHoh2v&>vN@Y+Ie?halX6bE%FtRzTR2nvJCiPfy_^J7#9~3m^!+@0nG~?;nTx zU>tP`IcikbnND8cZ;op?o7o00>RrW3N=eX*$Zg9maDkW$P(iO zQx$(P`M8`U>s$rKLa6lW2I(e;1U;0D>$mS7p7n3A`c#WGFLRts7^3h32qP0O?hauNM7J|yJrhMXRIOTVW%U8!CdQ90U5nq{F&R+To}ct1D>;# zJq5Mmwy8zRNtrhlkix3U39g>FF|MV)(Z}1=Q_kai&2t{`D9hv9Uift7G33lFGRE;o z?GdyOn=QUAI>5+v;&BAULq(p}J^f>mUG#x*p>?qYt6uX|#^B3GP0heuiaW$pQgnz^ za@6GAU-b5)t^#G4qx&*yx`g|T|9hLDc+B{N??8UAN_m%sl3oL~<;V-0N)e$e*`8rz z`nO(EZP|bqXa|2Mz4%~kn(~4h+tm*jy?*;Zu85K*t+C(U?~Yot-=T5Pe5Gr1!2A^&kKc!d2l@Zq~PX#o61{FQ_qMGbpvCuUvE3?fJPJT*o?u>bdFPHkXQbRWH z9`-3?H|ENH@RfDT@6!U(cuiop)g#>A5?8upz>eKlHxTb2K zINp=lajVqv4Q;m2u3qT524ki!5&D$u>7ABTNm8?s0>35cE~euT*{a(wXj-s4x}5Ti zP7$`b+94Fq#Ig88p}6_&aYLaSgce8F*YU8B;b6sK-vCD}a)Ym~1@;mhJ|*gT+`;V1TQl^d z8`rfnU*8U<;U=sg6?C`{jp|CYw_j{_oF#ggp|w+nK4wbWqC7$r60pj}PVb@=BF=epc2Q%->*cx5Pc<*^R4c;8ng?S)KGW7fgFo zeYcvdT_yPzw>31ld5ekaeQwE${8}XP%3sBC`=658%A8F~+U430e(@2A#NWw*m#lwq z*}Rn=-oNPlXXXYzj`*LRxk5=-Q_rsr&_8!CjXQ5jdpRk;bE`0;r@r}*!421F&4FGL zyIvp_ma{6swu3!AAxc3{NYdF@1g;@@Yd)XCB+4qE&q@LH^5ptgx54wS{o-irZ8Q5e zddkOUu4%d8r8jCGQ@5z?PoOs@FUlmIt7=bfcf1-lkf(40pDsP6E%Vl__MO`CVfxYD ziIezZFdknEO(524*u7OSW0PWZy}v*L3#XoY6~r}l=YnbQhoO=1R17D*Z<#cHFnbv@ z3#YzP77oDQ|Aj8%C&QUXb^a|cZc@=~aol}#uI}MfaH&`Ix2AylKFu=f+&RtwSwChG zS;-B1Tz`Aj{e623%bY`}gZ{Nm#V>B=bvK~X4&xFlY9Y;+n3RjW^98bG3Ae_i?Xg|V zWBJ)Twgg7i-WkRCMU?I1&ALa1f&;*I@pg5~kLw_yhQWlRjbzR(nNFkBRz;I&#MZ7n zeUy>8h<(8veD1+V#dUI3O=)-`eXY2gZ0O6PA1A7h3VwF?eywSEt9k9DJ7qBb#WeSv z0z*?I%^j({jxpJBw=CSdFnQx6?HMEO``qql7olEd`Ks9GZe`>bg9YbvV!v-eC}z5A zs3p8ygcJ|9VT;T1v=yQ`gRKS{T|fz-Wec7_N@{5*mc%>me(W{%~i@47aN^ml+LcYop(yR=XeptTzWi)*H`F`vs|oafzpl6Q_RC-0?(yE!|PA0 zqReN99Z&RqxHz6uy45uyyqvK*hTOW0`Zt$&^i5F5(4E{vcyst_f=}CGsIK{FkangG z+6(TXu1omVuPf{$&r!~OL8)#Z1g7ItfFn%MWf zw%?~6ZA4fl=(AR88v#A_)V>~D7wOwt@XNCv_eHz!>OSkjSn1pEcLbzugcoZQQx3CF zuy(Cu>mAw*Kkv{1Yd#0Y)Pc(}Iw!6j>wLpLR)_+xPEq6&?D*Fw;0E=n>vs54U!DT; zz}#zB#L%%^+aPQ>m`6aT(+-A*xWsei&;QYm2P2gQsqg0Ous_rl_V7)@rTeR|zIrgm z`2-B~OZr3~yLS!zu6fxDUhsm!t~+CsaTtjXw4n|Jd1n*E)izLua~}M+ahvDM+dbZ` zJZr}MviqgY;p<)P9|?7zjR#$M+#lTEy!A(reCoNeIjGZgl{FUXydF&GYU@P7Sar>s zsprRNsMGpw{k8du-}sJ@tM#03*cotn;t1l_r@_Z4EHKoCmQ&=>1%c5n@!LRu+Ol}^ z0K8b4Xg-A7Crc@Z&8p8IH*1`yMNNAV7Oq2tkkqxu z)<@02+z0Ks(&N|XVBKFe^d-jOezf%UyB!5-6JY`)7BFK9wePtp#X0RZG=NfG(*ejk z!U4Bq3Zxxyu5a>jaGf1vF35Hma>xwXo{1SIu~UF|}^Y0l)D|^O2yOHK@JP zT8nnPtQYgzx`+|L`*!U@3t1|eCx3sFRcIcx!=j7o*SEkI!)R?8kFyv zF%A5#^PLFLKk5~X6sN2H>Fe=YtPUE-dfz}DgFKmE%6~Vsg|?xJp2%Uvuj@~ z>@f)7t!M_Gif_fOorha^i*sb~C-vr{p0qs1xUPG~ivZuo^BCcCc1*3b`!=0nRQ~y# zKwErfRP(*i+WdBOB8lq#W@fbQf4sBa~lP^LE>z3m&cR!@$m2V z4ttmDyY#=Nq;L2QgM0^hHRU#C$0dI9Bs!Vz*#~tW_f2Y&p~pp|D`HC|wCE%10!WYk{frmN4BO zBO~Jn{?m(5}?FI8{OO4z|h65jgj2?e0HNP z@{iD!1AEks!B8IF*Sj3vrHz4h5LWaLBY|%XmB&Efopy5t^o^r_Q$KS+eAg&f4lSx( z<G#*oWKacir|pgRD<+DZlzJf?wUw9`1AiSD?*QM#n%nv&^Q&FVH8+^c6-oc6OkpQsqE zt{DsW=2us|`dRzAw0H~piC?(Zh$1g2D~HZ;L&x776pH3jXywJo=9G9Y-QLywi4hTx z5TdYqk0;?D=t`c4{^Y)znKz3Uxz%C|ZQc!5uVyklfwOqNDHvm1`*@El6Np*aF0 z1iFK7Ya4#7trT!)VF~D!ev!sK-o=|&A87}mUok+m`B8qO(^1TM66I_do@@90ANG;w zXg!W?g&mz$W}r{;Z{rj1-zet;1>G)rgWTtD-5$HgV+T9wlyC}rt&|5*)+zRnc=#g* z@dpa~lqmL1yQLo-I%i(^kM==5hb!A-Vj`2reJ1YxQP{-^EEr7P0wwUuI}1OsVC@zN z78@bQj@Dj#?X}T$#xA)X3f4bXLhoMovX>1O)NlO8Zwv$tZLp9j{Ga*EX9kNlfw3T` z9!Ka&o9$L06zXZALEVN2Bht>gz4h3+H+GO`qoE%E8qyPFjH`HlM^Fm7(v608*_(^8 z@>3_*dfr3bPum-HUzw*E69#2{s#oef7TWo4PjuC%MmxTCMPYYspI(c>g~@FR&;#G| zCV%}Z*bT75+>Y;?Ux#%G>3_klK|3z6@M}W?VM{qDopjR1INY1_&bNSd`ZmxjyZC~- zqFe1a>YY>V<(WYEORxix-Cp!Qv@rA#d;=MQiBcHa=^IGvS=+nDFKfj6hnm73PjCdCw!UPJ)@8Dmk-+Q-l@_V zN6Nn)F7t{2s~mC1gSyWk^;G{Tz3sjp4Rv1Qi66Q?TAkIYhG*mn^nxtm&`HnhK2P0i zq#bp8+V68OefXq&(e?n{0PD&-a9|~6#yA*b=_n~_U-BIZW7TbQkUaE>-^%w}n?QX% zQ@6xRoXoFw`t60f9bFhHe*G-%(Y_mRym8>oVh7uSX(Q#+1%05eLVFYZuIGWE%(%qY zia0oIE}{0^g3uLp3}2}Km85o|cR+KUHi+WsKHsnlGc&S3qoKHzV^?!YKjD2rLj9L*LFicwmy8NI1IAiXSI;*`VF&Gw z!XB4+{gv|a+m-%g6yc1^`z~>*kwp9-M5<-JaEu)y+Z;1GVt&NM(1w-B#c>giwm# z#ds$I5P=$9aW)6-`~qWW>}qtSuhE_ZVE)97nM;{B-yf8aM4hvngLZqhDNGwZASe?K2oaPD>lUNMR6XMc=t|LY zZNq!J4Zw}hH@o@k82J;5&<%4GJ>X70XU?3BN+$(53OlCm8^8~L_``#5($(w46HgqJ ztsKAYyhj*?w8a5yu$jAE(^p zRYTeu?f2TLpD8d-^+pFMOk^JS=!>i0JRf)&ac!o2@Ze#vbdXiX-#RxQ=0UrxHpZ&k z=Ah0aS38>Mkad7&kp=4a>PT)wx4gG$qCcQ(EVdNNx<42xPuJAd)wMN>Z~s2tPMnb*=As# zM`4frI@;VY=ItH!F5VZqyYtRFZSeY455`mN9In%Sl=cP`a0kpe5k3lfu2I~hu;9%rot{ACPu079b{c5WRci0$AA!|l$8k| z;v`tav7mYGDqa?Lf_Tc69>EZgU~X5wE?-;IZvRJ|UG^@zkKYeFB^j&*gYt(5Wae>vJQbM7?PzW9JLrm*5U1{6z>i1pmYP9P#5=P1| zX~Q&Jlc)Hj<(+t4?}C)OIY>Hr6P_t(n3r$B7_zK~yl*pyy&x0tiiO0~7 zW-vZk$LOgzUCLDQ)CUym$Sgem?6!xWr3`WXPI<|*o;x)t*yx${na>Tz0(wNr%6bM$ zhW?VDv3G6PCG|u&w|4EhoeK z=L_^oaP@hcI=PA`SZ{XI#rLkJiP}MCr9h`G3^^b z|JH5dqnoID5d>vy9; ze0is?gz&g(VRo$76wP*ibzjIMeUR}^d{;<0iQmpg`R=97yBhNFzDAu-${z`JUSN%z zFWN?^twR)c43H(f4&t^IZy|ZqS7pQ~AWb`+T_HP|%A&+LKuoYUVxh;7(MchBcLjN; ztc140P9UKW*$s$brVmor2*D5<=CIBPBGzgIf4c%v%y}!t!^kd6{=&{2&q?yhZ!{3F z(MIj)8o$h~hoZu6{<~5LeP1b$w0B#!+Zy0@kG@)`>S(Rvr)WbAPS8~k&w`8-0hz14 zJdE`<*IY9&40#uqS8lY{xc*7o^~+dDf0;+<1D;TFIrApt(+k0i|2iKITJlEvahB{f> zJa!metQYjg)wrsoF`-x)M~Va4jPl@gZb}NHtzAfEC|}vimA`uV4f;aeLYMs(FY(1q zJ>}!xv-Dx&`JK3dH1c+@Pw=RC48`4;Qs>amCUuKMhl%oxx?9@-a zpibjnyt;2(y^}Wml(vYU{KRqfO#1YTt31_HeIWJVF`!MrQzdmux|GvK@)XR6Y;vK` z*0E6YCuK%qwms>$lN@ zF*moxw}YSdE8jd4?80}QX%!tsz^56}JjdVS)?ORj8M4I_c*o>>_VLhWF8TRuShUr6 z8SnNOW-fubP%G^6FqZYse#*|(&UX%)d)UKvYk2uH%Fa89KkOb~A37H$+`Y6r&)T^> z^~p~wyX~^uMg_hdS04F>eW#f-%Wpj6*=4t#c-R@sM&Yk_*vHc`J+?oLOU5P)`~E2G zfC(Z%6!H9xjJ)fzLNq8HTu=}Si=if-?n4km6!Xq~`?vV@Gj-4gN>_HCQkVJ)Twd+; zBQkYfqao!4o`u<1tgPM2546#HeOk8(!dxiK-Y1QB1aae$GFK%3MxG5$j*T}>c_NKoIo1Sm7h82nX6~=FfQ^7V^VhN0-jNHC^+V-zCif3-43gK zGcQxVd-Yd_{M12w`Kfctcg;I#JsS{dl+-5yBaGSK+zIP=V!bX6C-Mc#+^rr+z|L2bzP?W=iX z&Qi1@Sf_1xgZJjX^{wstVJxK2^s98}VOwEW|Kw}!@{qH3J(CAo#(0J|q20>%UcS;g zhXNSt&Gpbeu;Um7fMLa2)(&wpmVw{$RS%#D=ws!2XC0t>ly|&AJ+cVi0_?oNS|w0+98d1`XW5ae2J36laaAdzo};o`r_~!kY0YtQ(0s( za{wLGuf_ol@=o3SR)6EHKI$Q!_|h34?a0{3&kid->bWwd%T*knO5z!BP?q|5ukVy8 zKUaOCZTpN{6*h#4L2t0W_?q*^If_40^VnA&~Gp&m4nBj0r-#Kyms(jG{Zb)_~O#e$PoH~ zhv_eQ>L1^tOUqM5eHhO0B;JC&)O-q(_G^cF2)xI%QyXnmgq}U`dCwcHW%;@?{`jV! zhePg! z@Oxkkn2h#Q@Mo4MKVi?Z>n#0AVQ;y_t5EKHOdMpgxDWHczY4pBDy{{dLw8b6yDSzl zCl_0!%BsuAA_(d-Kshzq?=2h)-%fG9`Al7uUkfM*0KqzK4g(ww?eZXXv0&wa`CwoK z3u!!0T!h>+@$(KdQoi^go_F$bAA%sQK10M2CS^$HdGZBmFdXA4Z*>Y&lGc0A7BBtdoi>HRxu(D5DNWj|4a%g* z7-MndAusuOmOfBM`prB2k^aqdeJ7s$Y8NYt6b7HVL;4t`P5m=}#Yx%v!*fu_v@3PV zvo=zW`AE2kfT4~#aLjIMt+4A4P?q`-WTf*<{ZqG;t3Sh>J(DJVB@b6|v@iKU@{pf+ z;;5%Q%?0CA&k6NTnjq~=KjnA*T%Lll9PPhigSf}5cXm0!_wRx^W1V}3fyUKP|0U>) z`q~cKGEyM;q|6!w3H>kByXrY~POv`eG(mPh>iU?Y*dbWt33{L$N~Z7KeLu?x zmECSM%E4CzZ97}$v`^s}n;6pw_IO$pNHpGf@fM&IsgwMJ)D;aQ>?@0_FGCxeh!#*l zJmX@oj*MlDhy-=@(|34Q+FV2WtRrv_6s34k5!yq)wBLLE%_C5`zR9Ju3lvp!Qk$$l zMoDoPoz(@6!OOIjk&jZR9Q|w8-uRe5>SYWlhUSpDz(}e;eHJ6G`r@ORZy>#KAQNPV z9Zkymw-d}@Xl$jWta(oPQr>jEU%0#!7p9Dul+Om1wZ@smCZCr^ne>(ZJJ%FzZ2c<6-j2(NT5g?ViJ<)bd{jR!e2ZBVv0s0U;{ z$Xg#WWMwB^IgA5foG8@#Pd`el4f4|up6P#i3$?;-2VVbLTQO*L!F%cJF$4Jq`aSbf zeQSkXf7QylXKmEayW&J)j}qSgEe^%p$?>jH+V7Lk28Dg7!PF-G%%K@&>SSX!rA&*$ zK2imHJn*O|PzWnxWdWI_CVV|P>&eQzUK$I> zy}HEumpXt&6kzNYx%d`cU3M4RW#wL75D*Fk%N=H;Pcc_S7Gt%r5j4aA6JtU2PC5bs zCd*<;nBbC_cIaDmBTR{p@o?{g^Oy zV}SV@ht$b4eL^rI^bibLkO_tQ)m+N=<<^q??NCa`VoX`goN-ORfM@1M+C>0VUxEY6 zblj{k8w$IA)kbMBAAPMq^@X%ttnC2kQ*j79+RdOMKZ2&Y$*3iM4sRroYqz;;z7SZA zz41|=b}5^n%+k%uO|Vl}bBrKv95U`%H}zT>3F;}GfB~Cd+Kf>X7_4Q=0|CO-c;!1j z^)rvfw=<}Xf;L5vv|b|U8XJ9MtO#y|6=nOK`D}d%*@f_2J+(!80)m~0dMA=#f~H#A z+M^wmbOI3A4Nn>MX};)Egb{fvPr1r5pV1FOW#UGNvfDwK5z=>pa?|f=k6`ZPAR_CS z5>nIrteemWb%45G>ha8Pm*5A|&=TI0QWl0F#==`;N$o%HoL?36F@%r|sA&&Bb~+%s?E>dI(|mRckD zp*5E@;bX37b_|``l6D06nK>PyKlIIS{Ykl_bcl=J68goj>zMN(YEUpJqw-AK+xK<< zD3`LDZodf{dDT+X}z>2o+> z)`vCVxADOv@%6;T57OV{8hk~X9At=g>7$Ihyt8T01$EU{V@@WBBC4;u+AMxtpldKX z;pu`oR3FOL2BQV}K^^sx`I7lnBcF^wcG%4y?adgE1anQO6-x4l`5;i(I>rdksI!}hF<8^_Im%vz-KCwd`!!Z7fN43*3ZfJKMl15M$^C5xZ zDjR?`&;mR$Y1Q4mYaQ?P{HDC`@rXSJJMV)N;u}SMP@kd6 zW!mJX{7PY;+AHi6AI2u*bAJ@}FdkQv7SXn79%8*9pU!Gv*)T5Mju!n>f~hIg$FtV1Pcw8%2HXDygSB z5pKN)T7pWeYg*5A0r;6Oy2t0J7q8;ELyZ^ zAUFx~Gah~$pL#rvTgFwqhmyiBj^EOtcMnwCtn(On(Abfb9`zduq1X5Y1=u*-Y4gtW z&?BF+QLI9HG6%*&-Dl{cHD2$qM`1U|%Db_q9T2ZR6YGT2r2-o|Ja?AX4s=vIl zjug=a{F)0!?CU zRxVy64r4!W8vIRr$fM@ExZ8$)ekn8#PiQK?7L=u>TzH>j)ptrzo2JzVkj zl&>!O6K!L>7tHBEy+Xq>Ze7s#wZabJ4VjaEn{)cM)=NCEk>{?yRe#=do>w0Cn8$5& ziI1Y59reI_C(1wnH-CHJE-!r)cFOrsbDc3|datnWNT?@J2+fu%?2uhIf_xO4k;b3E zD9`aI>+ScDGK2Q}EH1m)77)gfQz=s?*DzKK#}1-)OBVrjB%~bo+Mo=?HOvYyZp=-e z){Bz68ivDangj9D{}f18;SeXl^tDl0g!wFFgb`sSbcA7Oqk4#oxOYI9kF~Tg{bBZNeU2Eoz8HaFUHl7)4^~Nk>Py){|I8KHo)P=y)-X&0`T`r8OHYAug zj4#?)8&?<%#KlD8K@iAayV^TX1V-(Tf^WTP zGCxLwzD`izJV)~qYGI__sk08dLfQnGd*%{_CBghjyApV6w2fVW&QK%-N1O(%H z9KhHcpY~YhK69+Cu(#LjM8;?ed**A}8yZt98rvJ8k(3Q%L;Wged;CcGeTCWv$eNADHqJn4XJ9N))AhX!bkC9Rvc+d$Hs_g9R zZ+sH{(Kq$?;S@jPPGJu%mX0h#-eX`1?Up7U1e7qmG5Ju~1Y<%WC;th?N{FEtvgu%K z!zb}v{i}|~DLdmp77*WTfg3E zL&lLI#~czzTT>?rd*l}Km^D?e8}VJG5v(=wjFt7E4C8K-iqTKNJIUYrkG!j{;@cp~ zzpofu`RYq;4y~(Ek0Xd{oz|Nkp|SZ5>SN3@9}}mGXJ#ke_v%iBkHTIn=}X+g(ZKBnYAs7M~x5@#KUQjsZ)sNNT|z+p#m`v)5O>@91H|;vD23_DXoMIZn{y} z0VCFTh#F>*F+pSq-WYroMobvf)HY@0n=nL@H4%Zwa6Lm15FvezkWmmqYHsw@aZG~O`AkraD6gh-hKsYH7o@092N(qh7FAd8% zp#)>B-47`(EXn#N#sDCY86Ug;5dsJhQONW|6k3cxJcQP_Y_aC63x=#d=0v@_qfYX} zAPFV~dY?kx% zxiOTFzKt-FwG(7b1>#zV0%jSdfk2RxZ_V$nAbr+?zS8f~5#S>Hssjkxo*fx|6(fuP zUd~wSi_9D29j^rW69mkAib8|}0sxo(N4vTZ~t2Bnpoo50a02eP9mrxJX`kZ@rt>nWO0oeKQ);hx$x9bE<2H z-5TZDlIP+W3v&xCtQB_aR2oLI#J3i?z~jg0RLVk+qp;(96d%u($5@N!0lH@GhaW?D zA7z@tZtd1}WJnL)!W${uo|6O66bhqVJ;p!u1mBkzWk~(iQTx#OE`?njbu}h&??*54 zDG)_J25TDy>XjJvW@)l*ccx1R&>v}MA5U;jg|}eTrHDOPz7~$ zA6{m@;aSE_KN%wi3!6KEMqJMGC|F20&yb0 z)PXD$`JF*XzYFR?Mlcukbpn~q{B7rN?igpG=8cfP#Jg%kxR7z|LVGOD6ZOmHo%WNt zTtjCAzr_g+&VAzgJ$TkA=y8E}4WgvCzg=%*@IUVx6n4t_PSan0n2;d9 z)M+HhKUV^avT7w2!_kKV0#~~tW7?QHq7vpzv7qRLnd+Zd#?_5NNkNOi;T=VoGH9o> zw&r2Fh=0ACEnXM`CK>?*GtYNCgrO+T1PaP}2r~wh{0SWL<2Hu5rEGm^>=CTGFIc|y zlWU$Ky7G{o^*r^#lrdfeGeSRQLi+^_yxz^qPP#xKOCQ@oH;$C~%x!I>+;SndgG89} z-1vvlA?DewCZHH6%*`CqZ?U2iKNL^CD}ehG&7b6ah-WjI+b8JT7d4_ulDySL2|*#yr9x zVU7?O7f18Ym{0&1O#}j)Ijhf%2?a$P?LJv|m?{C48zOg3Zj}*oF?(Yst$GDq=FNXX zk9+SK8e+9qXU05n)Xxs0bn0j)hv3IB68CiI5>hlFP3niq@*Sx4f_S34Q+ z^tZrprhdvNxQ9+_pZ-!$^_9PVZqr6%_)t~Y(H@4nkv7QRvZ3>1u`Dn$j1ZQ_%D25LlQkQ7X*~>Ex>neQMJo>5V%XDf}Mq z!XNPhhVUq~ct*UIw9nc|8Bq?+b8-aU9>aQ&F~bv#L3j~JlS2jZbhI^jXMHHs`>cV{ zfPWb?Ms>0v*~OJy%Ihu$R`aD+*t2HId9}iB-bxFU$HX)4`l$};NuFTj!msfp+z2vVs2g=3;B`5%sKLpa-_E=(Oz``ZMVroX7M~5 z4Lm*K9i~qD4ai^QC21LEc#V+9)CFH7597DwT7hh4o|$*v#i)e8lhfiNk3VJ&cA-6v z`leRcZG`X&i9w^Lv#zz`?jG$h7c;j9@ir*xwGvJ__imS+9#)?H%x4e2SNFSC+MO!z z_dsFa;1a*{&NIt1pZc`&@L3OQ@|4F=nLadlw_`EthZ>Ac#wWz|{wVB;PYFwY_4wx= z!$nLGH1RR+F#VBGmlY^8sGsFteGpP@LEO8(>q|W`E>?Cy9_k~H(dyz}T=ggTXJ-!x z8J=6b2pmQnCQcz_-9vz*BvA@5E_GI43Q%?uC|%s`SS7>TTF zb*8LD@gy)93j~is8excFLct?gX#-~Dd5C6>l#SRRmddUbcEnacA(n}kcWvk+j6j{z zU)j|V^iRE;f}n;mBJ43}aKA#pDI$qG8p6adtq2nY1cH$K5p4JR#XHPLJ+)OI=~v?> zpD=xWYfK5!ep74-Fw(qq!|rqqc@pdxMEoX@YLoKitsJ{(-pPxQs$XNk=<3&uO_XZN zR1|H3oVlRg6!s|Dln+cb>yt}k<{Lp;Fdvh*_L{p*9GB@)4|^ z?DAQwQP?ANwu5<4e;=9uP$lSQtzpdiPN?%Cd}Zv%15Cav5MUAqO)z%^Yb?Q5!hEU^*&z@^9ngucHBR|AZ}nH3RvX7W^WL0)uqf>9Ihx#`d(0++ZHaPrucM4D>$%oeD27-tx~5yRjxC=ua>nWB@R~D98Fvy+=~m z$r3pE=GnSzbyc@_W0y!e`kUG?TZ!Qb!qW!Ln6xn1DSN`U;xt_kYzKyY<9@f6Pk+I|H zWV6NERbO#HJHWGq-1DGITIHrZbwL{h;}kmAhH=cjcGT+>FGLSAm-JWNXEj~*-27&s z&&I$YPR9=WKoM_#`?mdQPkDNI#KRvkh(Dsju8c-$pE+}SdGh0*Pe|zq8J@E(4>b3|d6u*NY%|)Ol|XQV)=i-^#I=M^o4>Cf3(_ zVb=L}A^9V0e%Fd5gVxRSn$9}T{S85pALbLK+HNYrgj-Oom8^EG zlb9S!B4E0N2)nfE#bTMf3DF_8+yrZdJwiN%G&}5=t2!`vAcnPqAuXYXWs=}QxB<)z zk<=Hp!j9pqPlP{xE&uE=srvXcLlPhv7d+26Tg&p)r{;oo>Qlg^t!n~c1UnYs zxOyAwDC`(U6n3zakaZ$&0TatJbBMw&uK7afh>(c+5d!tkNX)K{nyG4EZ7hoLC60H> z#2{U*GXh2oRo03*Ay{j%coTlIGi~1JU%^gG=D&J_Hm04f=B|9f?n6TI5V)O2NYVG| zCWr&djGMpw>;|a2c(oGX9S~Yl7tbSjQA!B1$=h5shpp#)3L$+IaLNNgThPDaD^owF z{>I%JLbs)pURw2vkSAT6X6Lu}XpK1}UFy6e$pND&>}}{1Mzy+q;j80CUH1I}I%4hB zZp}5i%C1q*F|^T|t`+v0m&Z*#V?Q2#3Hl5jqI9@s9RPkJj({fTIa-E~Di1x6l4h-< z*XBH)k##1X`emJ<2gt6UD(AA`?ifP4cdH?yOHrw?NSk&7rVso4G09dWW6D9>byC z*lYlhCFE}h)I2aB@Gf;tkWM}1>#9BV*h^1g&$sUN4!g0b$28CeG)FLIsSh5Cmnuhk z?aa879~n*bRobjf&(I&96YB1{G%;!z>v}T@f7HIn6XrKwrycs7Q7UnHImw%0#&0ke zjII8_M>8&hmppJ05?&+9`QQ|Iiu~kF&$-Dv zS>BamKUvw0!ak+BQqX6KHwk|qDl?`J%H6RT^+OGr4DK_T?ytg*c-KU&9{)^I!X)L> zLh8yl6Tkg@OQ9`X;_WiF>y&nThL{W5f*^F|RU>s27h$dm9w9+Kg7Oi`DBCQA2(v{U zt24r9XB_btSe>Kb6BaNp3R-r~5OoTpfIwhswct(Q$YE=k8pf}`#l_4c6d3R1L-+~7 z^FB(lz{M-flm(h#qde_bX3AB6?`md25yg-&!#u+*F+h1hf;tN78zG7HJTA=ov{u*& z9nyeZ-?#;`K$rOg~rFX`P!=`yD% zJeaDh@s*D>Z$DeKOh$PR_MNkA3Uk@6WqFfQ@(s=qh0E2IqV(LVS3>rJg= z0095=NklT+qx2C0Bc2QDp?^F#?+A9%YLgwRtX)PHWfBmxi^X`P@52<$)y%I5 zOcCs|wnu~Yq|WkE2kwsq2v&cFu(*md9>m~7vCwt~nHYMMO<)krKgNojs2-O(b5-8j zMM;SOsGkX#b}ZzdwaCDx9>#@mk@=&ZAm~%;-ENM$8wY{FrS6dLL5(qkhw`;4-_3*F zILf^87+iosD@Fx|pcozqKX&la2N8O-g956)+-I!`!F;nGl?l>;xXL5AOQTHv;}ZyT zNL#hX7-hW7`HZbN4+VwYbFek5sk z6?SoQwGPuiS=;H8E_k0^L~gFUf~+^nSMtRt^sP0;=%2YmUNAS+DQg?h7=~6f)B3Sa zDdp%g<8j7>aTaYQcqcsrskZrage-#I)bm<>(R6FNHsU((0`C@AeLU0uWC!up9X-!k zCisQ5W?jhFd_jx!MdT~AT3Ymzu{|<@^&NwwIP$iB<7Gt=%4d8Gy)j1U7kn=KD7&#> zZ7N5=8$<8(kvi(5tX)_A&NG208(L({%rAW|e{IS<(1#TEC@Sg^k1}!Wvd7CQ#uRc7 zSqELIH&xV&Qil$aJz~TL8zcIQOc%PWOfrH#GH;`>%ZH1%V9e0Al+BoGUTCA=<{`NR z&Bn9hZ3uXQI%-dNu>NQ0u$!)rvkqN7(?6L{UC7#itPTBVUfHE)5T=;Nt(#np&*ER% z=*b$%y}ltk`IN(sf85#w>y8{^ofuc~a$kF5h>Kqu8|yU?&s92cv{k=ZSMsK;<8R`S zA*{(5eZ|YYHkm^L1B-U4hjoO1kzE-?(mpbV^7J>^Qu*rV%1B{eOB?wZkF*X2^F938 zc-w4{P93cuS8GAvX+vD}$+=`rvXNkn%)5+Tdpyw$<87X4TXxvBA^qgH`^**3YUEkk zo#zzxDB<<@>e+3ls88{GgTkI2cn8i=+Nb*6sn57m zVlXT1Rc~cdU=e!?oW4QiqfBAU+`iOF|58%3d#;}d>Jf@EUods;4CBbUBEV^r_u{Iv z`#NZYd22^Wnv{bfnL`;H^H<*xB;^qSU4Ii0wTU3BZz*)rW4r`1eV!e5f?td>m{J&w zc4PVxVhOboa_oqyzxf!!Q5)UcY1AhKj+DW$q`xD)j3&SY@>gyY!~|=zRv^ZLHJ$lH zkc*%yPyH`YuJh?hjoMg~L)knxuh6)J?J>;!cdzXal^yoPspm(az2illZcC6n z%nNj^Z7?4%Vpue0j5zYDX{|V}cspZM`X^)Wek6#C9t*W$JHR*5D0!h3=4bd#8`iZe z9)d1gr%{k8>EXeE=7&e1b@3vqp})-Sx;@soHc3Y@jFQHK39S^DfS)|@6}$;Oz+0j` zQ~J>2D6M1*{Sd()l;^h~Zg{i0<0aOVc>0BM6T>RmCEsVG1L+qAXUc1MQ+>V-KJB@B zGDb#Dqd-~r=m`FkbqnI?)6f)s$UrC`_0~4!=qu@S28$ibxWuDd0vRd!29!W_Nj_ww z%rnR)M?m4nx78JGPn{T2&~ndthJ;3%&)HFyKOfp)uEt6I%|C6;deSe(T%Pzd zqrEi7R$kJjUy`SCz?#xO;syFWb3{6GhB3)pHOI(Y>LIRp$`UtqX5cWF1^gnLGnD0c z{gHVhyOA#vpHazrmuKp(e#(o%u~ygxeTxU7`%I=SRoH_N(wca6%>mXwlUo|s z+-LIFd2TL}2UtYvRJ+L4$O55Q*a5SswYy%Jqd`3VWszc%`F0BQDIq#Lnn2;haHK_q zqkz|aRQDZXiLoO7K%ozF0fHhW4U<+LL75__K5?5um=Hw-+jB%KJD`-nxPu!51T%Rf z=K741gjm*!q&yKth%iHtK(zH=y~9p<*G~)r4gtz@Z9_Z}%ealR@)81!MRst-iSi^L zmht4lIHX+&sy1efSw{&KT%qG>00=cff#xkUxDg8V?uYg=@W|WRsohqMt2%o}_^OpY0%pb$w4KnPP3D%db4~k4f_Td9QhXyM zchQtKv?Jf2DOdZ=bNRRz2nYGh0WCfO`DURqXQ9K%ymkQpbqj9%$fWvKn+H|v`^WBH14&- zy*Y1;GG3vB?!~bVv%??0GM@38C>ZY!yY;T!=1G*Ov@>&QJgCdQKj2cX&ek~k>{(ZN zBf*>j_xLmVSu5F6bw51)eE(*3mQy0p7)v1Md(I4V8w=E#);z zC3?me7OBTqNp?a<3u;ze&$yxKSMS@=-&?(OcXfm!3HffpWwvm|~GrWc>QSmnN zllj&xT#1`*X6LLQsO*F;c8RN{8HrXra+UlD-_qD5Ixp8RXxz)!Klx3U-fo_3^-TiG z*`-bM$4)HZSQQLb2DyFCF8t%TuPIoX=Vhojag)%RQqnQoj`c- zJd4Jblp!nAL*bYTx|Ey-JRwziU`zH@NG~K`G+xG0w76ORQJcbVVf){=ld71w2J(v> zZ#5_0R~ZCGvn1)8J;~8c+F18ZxnHlB#2YB&gBdW4&AO!g#O&&ZSrqc&hFvuF$rl+H zq1zbhlQm~v8|S!0{NPaf!9zRoKNT6ZudUN8B;_1V$*UA%hHtvP1l@WbIa-gAH-BWBY_vd+(q zi!E)y(93uDozybe^}jXGO=g}7Go}~$k1h$&u0=jYi8T=&P}R*F7-}u8af-wX>iL;q zkE@$l?5Svb7c))x;jNCqg<4*evsf?J02!Wyg}mtR=poP>5~)#kOCTOOlJOWl$u*l* z^nQ7rs`EmJ1e9VthT;b}#mYEm-7uM1P3z>9hWXbJT}!Dp1g~3c`*@&AYz9p*iT!>R zG#6l)n54{_9#rML`RX_p|8cQK*7c3*1v|CZ%jRBbonb$(Q0jWHvkt4OvFvrp^7%{r z$46UINfiyNJ}~@X!1=ej)z%Gwqy^0(NaUtBDel%UDzO=#(jm8%H&7RP`q(;7so?&e zkbEL93EkiCL`saGKvDb{@@xH5UW=r|kgF%0g=QPtTnuYFjxADD4I`=ku#fu=QOc1I zuS>1K2BLN>5511)<6gbudPBEY|KQ}WnqYs03IB=XO4auwm9-3pRiM6 ziTuaF^z~XjrHpZJ;WnPYfIOt}T!EflR7h1DQ8N24tN8-jsL{r}P{UWQxUUbdzDU6d zv4z&)!3=%~iKvySK{NQ5Toze7;r>^a=sLzC0Dcp^G2i-i9f*YZLqAFA@(0X@&fP15 zVG?zxMGakx48mEObSK8oewv+$N7#7`qq%!<2vN4O?qctC%737GZ(=tUw#dh=?o%Zu_v%0Gq#%hhLw?cfC1G8^=_Ax(~ zzXf2*UDxcaNgooo$)(%i7<5?SeU%ah-l;yA1eW(xeRmIQsWsf+H~voA@-u_CDl~m+ zuEBlW`d;DF!ZbDLg&JZGnNe~54)K(Dcj3a^Yp}%NxPOkr?mq; zFE<}#T>NMwSr1?RWRdcg#*ocRS%dc*wKo^$PkErwP{W^!IyUmS4$!AOm6z>zw4sRj z_-CYM>uxP^?R5+X!;B~6z!Qm&_?KB5d#zO)wBZbh5$B8I!WPSE8wKJgsGNK^YmDmf zXy(AXm&k;>hg|}EEzgT_pIg-bxhjy-*pc2=e1YqOlA|q$q{&Cz?;-)K& ziqfYVP0o}(Ha4?FGv&H9b2G)Nu2G(1k$D1m`L4@<%_mV{q>Ng^ll1B0q2<@~EhG9a zcU`~n(|=9<7ZKO|w>t~%8XgqccK=(TqnJ);8NUJser_0 zp~R1w=|w43^~@G2ZJa^wmTtHQc+IthJyTR4BiMF~d!TeAdhuvLPs!*D+-UajI8hE~ z3Q2j~V5S3xD|&<%y#dbr!dXs>9avaR2?o#)aBpDiuWH1c)oeoZIiBe%_jv1HPDCEZ zDqR6n+e@3!tt}|1)u86TmBysMrfqUOZ4F5ZF%>k}R6CB|qeW~=!R~QI%On~7tQ594 z*=Eyb{?1y{jQYC^y{z*RtgbG5hnoYmVkyjDJz!XGuggNPYvy5Zs=6lov>q}#%_0*dOW4Bc$?OLAHnj;6Y3>e#>*8~$OOr$Tp z?7o|s^}UmVn$WL(EnL-c7PRZOEd$Oc06(VF%w?du|9bwi_tiKcr#eg}j+QDQY><{i zL$+f|n{4G~U-4wgF|DNiau>J8NGEt%f&Y}@x8Cb;>t^%c153YzDj#9(Ya>nN7j=?#lq+Edk)!SLuxB9=PJhj>FuIOIz30W94?EoU^jTLj@2%BNgSU|@&$u!9aKXc?Oys_Qr8|v9| z^orXzP5x(@0t}giJ z!P@j&ndxrLv=l|Ffs*saP9q4M=g)hw23;+QuE-vn=s|d$T&r(oFItH#Mc5* zb_~8)YU=RX;8Km<>X2p3Y*{Z4|BEY0$?{0a;|CP~qz&!N{s?(xz@}@x;)7Q@d_5}= z=-v9mtxp!44DmVFxB3lAYJ#?3X)hFt`!YA7+N}7UQ<4MqT5-c}9`i?n-eic^*eQ93 zNBXn~7m(@E=J!zVjFWg%lh4f>*X9osTV+)5gT6f;)wPY|5E5q#xB0QteTxb9nX!(t zs@$olxXb>;)p{Y)@Z-b6e`Of_Fm-E{Ja5fHUNBJ7DK^pPUpD<1kib{iuj>~cxa#0r zpi*SDQ@T>~_ZXy+9?6}sbt;@t@#YiQo@93j(DuraZ{Rka=ciTIMA&JnzVhv3gu8K9 z8U8~rB2prdmTv%U+g-R^_Sw{Wq`P%S(E39U;QxQybd_N%xk=6>JX^Dqfz!+=K-kR6*Xx>>D0cEy8m$UBgc*?Mmk;*FWrUWy+eoEiLj*t6ro?1xC> z!<_qSY9=}Xh01YvmE1i+5UbhUE@;q+xrpU7RWmLwBOkW*!qIU5cOTV9a;}6mmf@Y6{u9eQb zRn}^G4RG{2ivxNun5V#~wl6+8QE2e&eio0nK4)L(NfAw+=L#K&nXa)zYX1sEr1G4% zs9_fV%a9#J`ZAVGeBZM$Q73~3P@(Gq8gmy!+hr~}+xDDV`exCoXp!{UekC$oTPild zb?v-Q*H%6DIFC3dCO9Fnaa$#j(kd%uC|z;8;fsTR4P}=-32niE)qW*(k>iT?Wl?7` z3M=2nuF`jhwJl1u#Be?sP4&Seru7`($wGk<`{t6}+$SPFvl;n}M(d=&tK{-6JyP1A zkHvIOR$$K{6nTeyNZ(Doy3-(@Shd=CIRK#&efZ&%<}Fkjta2TVI&@oVZdskg3PoA} zt>L&0(;MPNw&inx*5`3Bb}<2LL!iF=ZeK5ax#!X1S^C@J{wJZ+ga$EDr#j7^t(LQ~ z_LI+WKm)`xO#N$$V?ovd?sa13*37`#fSbcdiK^v!BHQiKp)xFAegpnvR@QeV+agH< z5>2(l;?jJjz86u;?!impk+lgg-?O`eyK~~qcY&@~Vr&@~N|)H2fM?LNqr4qO*{Dz^TIx69~DWebKphz&+@-BBIAI@Falwhp6-T z9h%o;c_%(K(z|S*U)bs+paSVZz~4J7N6|2oKN5B)Ol7l5&~WG|6H0-$#yQWeccH*E zdq=GVMrZKKT16;^htOi~aDeiNUNKdG4(EsrEKJgF$~9X{JH_8GlDeAEnRzuT2eR5+ zD6Ka+$>uXpXU;4_i?DG1_CtDYw*YRoo`KX0BcXjh-u0fzSpIJ^ zt6D12F&0m`_A$vcE(4b%13Tf%3}g&JzR`|PnbqcWEK|xim>uQcZpLlB)j@rad3X`35?}-in%724!U94!7w3G0B$$n(?g%OA*FuHftmQB={{Hqm-2tt_&4P zuX7OtHT+y4jEJtzjdP*}``Uns-@M}PezOx6L?5x_?O9kur<&!-9T9Ng;~HP3;-GDk z<~HqvEZN1wkwO)h1B0kaAqXU*-tz4xvgXflYaloypd0SlaZ);@GFu@+zF1ku;nN+- zdCJAeOz5a~SJo4>nq6k)MbJ=(of$Bko~-NRb%HnbUtaM|u@^j4UoFHbh_qQKemtPO zJ11J}@0s*pe9ir}wlML^=TB_B4VTZDG3--_4Q!0Aq%eKVTO$TZteSF9Mft6|h|!cT ztF(5^AshQ&6Dk-#RgN_IR0*>4s4Iw~c7p@5N9!m%1zkZ8>5L>|`Hk;I&$tXk=OKUH^^W2NM4| zjU!;Pb_VY&*W2G;6>d;N7jEa8lakT+(NV_ug5S_G4mn)GM+x3q3pXa82JYawnqq%?x0;Z;U=1 zFj4dHkX}rd*YjQg8)}I-$xWZsBof=f3F2-03NJn?;}0-uht8Q~QWh~N?9_dx4i9N~ zNHMRj_8Qnib=>O0jDzK1b(7rUmE6$OH~v5Z|^iQas%}Vz-n?t$=Ayi z=gR4*n+#2%fWK@6?8kIjPg2ADFoN#gL+bXp-m$yGNYQ3DYQff$dQZV4 zODv^IihI&b>IBx>g247%)!_mhuRr021c1F*Hv)J>%`}=HuesZwS_Hy}0VP%x&4rc^ za#9RChb>3^Q5a!22Y2e{S;y4rLrxxBACv>Zr$+zH5gAI|FgXxd^8ens0uX$e_iMQO#GMr}~C@_W(IY23!Oaw@?7E=TJK)O$HD7r9y1lkTbP##GDea;+D5 zNQ>-hm}PR(;_a1OSxd0>+BQ?${cPxz*@zxikpS^`zPieN$!6?XSwMnF*9qy$)ihd9 zrP>$FPf>%dtAwq|>{p8`PKQ<<#FV+Qn*sAVS$5)RkF!IV4Jizh6NA63bEjA+1dosr zi41lo$NAC_<3c#zGCwlh9=w6qdWpD&NRA|%=ClqK^uTgW+72I6Rr~KwL0vHj+FU=y z<&S7?#G2g`uEJ+`xv{*6^=gS{6Y2+2;FS|6i#T$pY1>Y>mb zQ^<9Nx%G0uQWY+DCoSc=!bZku4lRqIYs9yB&eJkO=xp%b7p`o>T#R)8cY88*Tev9MzPnYhb zhz{e4Gy4EHLLb(6x$r!H$L&nw#0ScC;glUvbV4^b79T}S6^N2FO{mNn)weF)h)y{l zHCvN}IIicIMJqmNnyA~6W$R0><8<^IdxIt{S4WZezM(Eo-)ghG74OY^>^`7s;nHxy zfYKY?@b}&uLo0puz8cMSZ&0%Ri%;k>pdUzxTB&$n?bEtb<*t z{zhwacl{e~jPrm?r8K2Tnh+MKp#asE4C5s!#Vw_cp8E%K7Y#o%TBbIBaqP_mJ`?AG z#L46?zkpg>nAky!XV1cvpwDKVN3QaVFI7*X^u_$zw%?%Qxm6m6tq*^BZcLZLdCkfN zykM55fEG_@`F!KKvu2>V{7Ta=htpbuWSxZxLxa#&o--q6sdsIzzJdqy02l?!e_V`= z8KnD~90WsDE{Gy2dgy2Bq!$VG4<4aP|9=-HND|`Sz5e$<%yN7PQ6;v!PqoyZe-qf{ z6QM_#MEd^4n3plg?OfF}Yg8^Eeja!i2#usY1W!B6AettWSd7oyn(3Ox4D!$?mgU&~ zupVw|E9bDL1U|d6^OH*vb+a5D4MmY(d@bv5<$V=t z25}?Bp|)O43r{tGwkV*Rhga*LMr-#d%D{Xpjgw3PS+Us4`#~4 zd3s8#&#$M+uRXcX_rQUd)=ByAArDdc0b<}mJu>hPqoU|$`4V+##@F2bVAg_{{)+cV zpQ#5NC$?W{UQQ#WjyeEQfxP0zGrTobT=I28IFrrpP^s92P*N7CmN^@bo5!08`4v>93kOiUL6MnRmqeM^}NV9Be&qo%luO7 zoQ{;n_$WUio45au&!bPuuxY|oL_l@R7umNtWtJ~Y@BpfMCFb>3#;HinfWH*R=%lJ$ zlpz#YX;)vIPwMhg?QF6^Tn>I}nUF@RAUAz_cS37G*<&(KOD!&0wK#$*jwtqq-L7>7 zk&F4lHG~nd(Batfl1fL%rE9ep(Ci^EIN7db>+sbQIu*h#)h=v#UF*46>%#8ff1>t6 z^#Fq+8E*XOkm1K!Cg4@^&N89ZLtlQ~f*HaG3HsHtzZKz7e|uae>AcBPZ}t4 z=k9BjW&)}OKfLm6rtCJ92Yj#{e;pYn?CBofEkxlb9MSf|EOD1-iYHsQlp|ai=>~WL zu9lmpE(%1y96;Qf!F2+0pCCtSnmO~j&Hr0I`J?UHW{c!P$1d@(o8s}*eqBU=tS@bR zB&;SYtUSYv8FST7ST#*{OVluV>aZUpBM1PAnS$BNrznBjdV`Qn>Ezp?#E4w+ZaH+u zZMwGG52zK@m-g`dhAxj^ju*`RU!MKadbi)P=BZ!f_k#67)MaBGMY>-KkZiE0pwcg) zfKe2OJe|LHZpHT{Y+iqG#$fNkSie97>HQ#8^2Hraf8D}Te9d%d_$$!OZ- zLVR`0^$1C=v#Rq?mX)=15p+hOKEd$9@s>PXvVQA1XY=M2OXT-hORr%4zY}Kd-*=tI zhxf1|-U12d*xaIgt^CP-?Rh&LLssPH;C};aZLT9f5fU#8M^tjUAP3J}W)fuDI_2U{ z`$1X$jga+N97;5i><+|>2w78|NWMmbOP19nz<9JZS)v+urM5yqGCjMaLz*42UCf}P zVrG>`NXuWzxk`v?eq>Rh<_98*pkFnFHc!xr&rM9&?`cE)m{ngi%ZxI?WbV&`Y#ci} zoeosuI*vwz1US`^Y^CFUlZwYioP(Sfk(`li^Xw=^Do{M$DxTw+?0{osXq*ko;`1bhFVI^2p6Wg@o6hiOMYcO71iMe6QG7!5sF@(=JMNp zM?;rxa_!mf%d>D5Zk1n3n_BJD0vL9@lamo*6@HqYKJdY#UHbm1xW`5M7SQS8R~@HH z>Pmf$9{}wFYs62X|`BxsMzz@{uvfP1Lz+ZPd%e4b8w@ za4v-jQn~V#e46buTbM*P0_EQ318sc7|1CxWaenVgM^n)?J$EIOzF$w%fN_yx*|DkC@OLL}NF9@g0}P0fFo~dC3gFr}fRZ{bup9_+Nk-vI41>e(Zgm z2sG@e;r0&*vISuoUn0dLz?ADsdA(rzOiK4=n|zkddy$|=`rIBp8U^+J6e+W%HG`iNWu%)3t zK5|;$=A2jjK~RzFKf`yLJFuNz-Z{_wRcl|Y*K2N)k;C%weq(Dffe&>0#(?XWK`G!#mB{W4~raktbMuc`XJ18**i z2}+&$PaV9#;atQ6&I2dI{-@i7_X+RbWwM9Qostr@b}tc4pEbOIv82y+8XuHIl)M5z z=OIG8W)3BN9}1#ZOO{>xt!z}!pj4e4X*7^UpJ{bUZ=efSB%a{sD-hC8bdB{D-E%r} zcFA|)N(cwO+hI$7V7dOI|NWOJi*IW?pRVzy2|fTlRl7dl&C#6p1drb5A;jyp;>{D5 z$5lHV^`6K5`vM2USwaU#DZR9l;w#yRXKdVWloHfyVqjH=! zjlcs+p$RD(`nnCbdgY+Qk7j0<6gK|{7Y?0D2;8!?u$yfgiTK-nxtK*w;J)clL}g7f zZz>_k$yZlXLDjy4W=koI%c)g6XpF5&))9?OxTn{ALoqdiUIPZUrP=Y9tjo7Nl63Iq z_0ZAS9Qxm*R$iBLLpm4#S#N3&K0jx=^_0-s)lm-IEH!HGi2~MzpjU`Am`)U)XOwpr zL~+e$%<2vx6V~2Xsk4IqHel8dCchKUKP>h-kH$_;Zi-M!5U+S$q%9|?usu$~P9|=- zo2(Y-yA?TbrP~v;CMB4H}Mrl;51amyPo7` zYYEP1a~p$$Co`vAL#`Y$e74s-#~k$FL=aKIYB3wrO`F#D`ITY~9(%UX>!cw_TJo2M z%52}0Oi8R4x2PDw#UYvq!mP_Y#Z2zb7&am}7C@8s#1E5u1HN_7{5MDPQi3;Z%);=! zgAk&VIh-L)%v9lUG3)q&T8`N_j)CMAJ%XJ-+t%Y_$nStg9RZ3zqyZNUPQpDh7)gEw zjLJ6Q`NgGN_qb_uRo4kx3nL7VDDETix*1taVMoHe}EROR<$f?qS#XB&)`*nP=e7PvFxz`{i6EPFU%}CJW|O^hx#t;Zr>nu?-Io#`&SKb7C7Uf=?XLeWEbBd6;Z|e zj?~1fL-Vi3te{liH9eZvRWHyB%i+EEI(Ckv{na&t>f-?Pqgbh@dgvHKb1}HXRuSE| z7))M#xd`QhpSxmwkK*Jaep zn>(?K>z8tjUx4brFL+y>*3Ozpx6#aZT!)tku?p3T1If!>L|P*{pqvAA*oabw$TawS z{Hr$}rfc#9XUwjwxWR;VESrCG_)P!t+^l6zV>AtHz3L1NfOqFzZ%BG?8covmQQLgM zpx8kU!%i?mc9zTkiwJlxF3;A0Or!Ekhep3D-UOBi$Ld}HffOUaLCz`LRrdnN$JOFH zT5D|2?R1#5U(f9b$#PIzD;L!_)EN6!vxGI>Fmih5ef{4F$F*?J>d5_3&1BvE)7?r& znyZLHHHqhi;-?e>lmG;k$QH3y%7n;N+PSYz*jPA>f<@*`4$=6akUP}GIXSWmwoh#4 z>3x#TlYD+Ms$jc&dO7|&6!g)gMk`KGQ zu$t_Y?tSWGzjvqjb9WXSyKe_agM+`!y!-}4`&3lAX-uq%@u+%rN_oY0yc9oG#Q?-< zD6*84cS)mNtaaHPKEdQ++m6 zlHGa7=XZOVH@bIp%+K$OacNs%$Am(zL_`oAFUzgGpK-OLLLQ6EC#bIR(NM#J&(0TK zd?i5|p=*c|KlmHDRkeSeALgrFI`8ADH7LLGac*xH{ z{2LcID_TtOG8;_3>%_*n44_zqE#OO%zx!#Lud0?voj z^nasQzWc_Tt)v$hUltMGm%hhyXdcX(c_PgLp@W&T;F19wJKc< zJ9vBC@BgPtqok3hmh>V6K(z+fQ9mlTBhcOX@6dmk3h`4`A4sLwtfVJQr%=r6en&XB ztl?G6|Fs41kAV_SG0b0ZG{q%ZO$!CgI=8=6K@SKc*dnl0diykFYgeF6QP8k-!1FtO z7AKDz&W2eum$UiRbMq z`fDoEeyvbBFm!2EV3%fyX=0ohztS-_Cd&1i7;nrT#96s`pXEIxe_qyOqu1m42TyG| z=r^!=21@Mlbw>xU`+*%n-HxBc-J_TfX}a^~Jp}eZe}Z**! z=6;5VL2_L6w%-`rrLtDWwl*j4iyjbL%Gl$a^{L zs3JxG*@T<;J*ll|arj|f7r>ka$GcTUHACe`8>LH+Ykc(1eS(L=+pq59g75LxU+4G) z^1ZK2ti&N|L~$KzgIDMMsw$u_BF)gZ)l+$dXy-8WmKGB*QURJ(o$bY1)7;?{HF$Z+ zo(4Sbe{on4M5OS@O>v^uMb6BVIF-@0x{wz5bjnICxq z)$?UgDk9$;Q#aASiE}y^5|1fWdR*nq1@Q?kW zr~d2Jn&4WtM;|K*pBUE$xBk5rMu9vPBPJap>T+}m|LTZC$ym-|X4I5Ov=i4erD|LG{%Ty11Ep*}G$?<;85 zRUMb)-%cCfD!R|yS2{8X>J3)7S3U%aIjhdQ*8As&ouz@-gsGHoCvFqgRqKVaJ&C|x z7UnZ=$` zi&?);X#eVPv6700*tgvheAZhH@-lNfkSp- z9_m=NchLIEqt=o-9!Z9@s>+Gq<^Srqlzl(Hw_aCQv?{M`?h9xk-0V!?5s-o0fEU~a z;KWC|A?Ah&{%4OXxPL}okY_nF-OkCO0BNuU-1YvdYn?m$EuZA(rGe$Ki;}u!h}!dp zKS!=%AyU?q-Z!Le0=t+Aq1`h>2L0QC{s6q_b*himmamP_%$ZJO* z-2=*b5}Sa>M6$eYmlVo4wXYmw-Ay0AduXO|hddj8%{8c# z3o9&uxg>Ha%g|Z5d?JXP2vG{su!bHBBw+U^N2R}_E_oP#Pfu**_r$x_{>>ZVj2dL` zek9N46n=MW+{-o*HAgohgh=$i)-L2|c4~0<(k*@EF6j30#)vgMjitAZQiZ}7D0#0y zqqRwDU9N3=Y%H0(VyhHyJpStC?vKCP(^y}0;a1pUCDOFm2uXvB5gWs8S5hEJOH2i3 zbFzZF^I7uSO^=H&i?7Retm0c-g!(L~31Q1k`*e@AcP95DeEoN7t3<}$1SUo&o}N<@ z5FCs`qCsRFw|xf0_={*i2nmg<|S{#0n|i8o3)4{s~KdZmThqGnF*o)vi-Wz|x|;XC*v zr1>&b`$c_W;Pb-bysy1~aNkSC1i0UV4gR51)`#5oum4DT*RL9`u#l29pTRShJGOCt zyOy~JX{~U@Hh)w3HT-?l7`9CCa;dUx*JlWf%0adlx9nXqr~IqW7P$^!N7}#VP5yU% zYC5M`Chi;hy;pZ@xZ*i-n1_!q<>_i)@N-09w|avxbF@rlyW zuoQVHNz)r{0kg2b>O6Oh4yLn&noIp_vL zX;iQ6%1dhi-S5SI`V}w`N=QmC*o>v=&Hcw~_5|YHJUbuG-@MkjCfO9b&bl0bD_*y5 zHQ2xE^_4irdw$p{PwI0`glvu{rs?y4DY6H=4m_6?k7V*Jshg@nX95*81iQ5ol~(eRd$b+YN1#jgf!l%!6gsZ|1rlz-_3`6RbSf z{Au9WwzmN5i|NiY3^lvM^;ql!NCZUKCFS(j2=*~*AT}+%zYDh zn`mwEac}i@>JUHZ;9FR<@P^>@^*h(uw*&^DsE3xhm40&s4*v996WM{=$yNw@wYN6y z`c2}hup%yS#da^di%E-7G3qav`=^{#8ASxLfc^g}9A;OU|Li}9n>m+h9QWp?~oMds{+4*PSn`Iq>#gTSJbMC|B>_hU^bQvhrQa&RDdr65x zhHS50!SQ4IFMUWkVq|zby>c}uQ}&DekeAE#tIn(}?6{Ule+}vYx7quS1UYGZdL--R zEq8l$B-_OML{u(&i@>9i#`yC;6xf|N5!!Wj1WjM7!RpWlTI&$0=(VO5>5t^?8 zrp!5zjV3?GEt^kQIcsIvbBhu%zjIcs6{q?~o7(cXX1!y&a`Eo9&@F1$h*lN+!juZPlw-COV9b@8-R*QcB<~t)lVY}aJ-_OIOgVd8l5)C9NcV&vBLlyH z$L+j^3WP>Qb*$=s8;_c$+>L*)au(`Erih8MCLA?^<5ix=O%AB3AyWvQWuP?}3yy^| zF{pyxAWw0$YA_W6|iq%4dQcU=XC(wO}S9%b}I}XSla$Jq2=OA z+t|ip)4IKiU|w*@xGtY2RJ1DuFM}>yaQj&^Y5$PcW(Jr0Pra9VNc~=YRx+Yed~Q@} zOS!X`b0Yf4H6J$EfBp8*?|4f>!bpfF7+`co`gMRv`tJlu-*CW4!L!hY6gy(wbkK?q zJHyM}8ODSwRpENisPPwJfY)LabXCr+N%= zg*C&MPo1cO;>vZ+`II^b*o>EtUQ!@o#XNB-9EI+H1MBn7w#Mz>7g(>){OOn!idj$N zQDWGGn~Ns3Yu+EjDd;mFh!+5(=<@brklIQuS4)C7699wgM1Br?AJ7mX?6UIE3sxR| z@T@P9;~BGqRY!ewciUqsMUa7AJJfjY_5$55P;uT(OVym?jVv*AJwK_=wYHY?L*n)k z0WBOJt2oKqF2cj3cW7uaHZAQgMA)sejKhC$!V~1B+ywTS?-d$ygteA&T&5yYThY4~ zx#x-qArl|g)8H^c;FROD0Oi8Q{i&(T!{#){RBX?f_(V8KiynioE7wT%A>QT=$Qy$atI<~%n6&y-lRXUs#20L4{a^sm*Yj29LdLxIuc`e z=9VfMyY5dKXoa)u-dL_pzbQ^V8Ji)rv{WbDvSJTYa-^qDVE;Rt$jxUjM8x+Mu7t@x zky$&j+p%WbC)K%d@YNw2K+x3!;M^g1%W!+~itN@=R|K`}(L8aj9F zBk7OnQOoxI%@urlM3ywWuzXq<@=CO!FMi2@0J7cdT@VMM9Ue&FxUXo$+`Xb?mV&fS?QDU`s2yX1c zoJw$zSQuDb)kdF zBleU$2puYUX@RB{!o-ILj)}MoWoMbCYP!9%O1Pf6i`ptROEsxzEFQk>vq?cNxY=5= zeECX{ThT?8j7jqvcI+?MK@aSHaWc>UKf682oBcVV((6?i;Y)6-M=aYaj`?oOtiu0i zcC~6qv^g?a{keiZ1(+QU%t7=EYa&aYlUZx6r_tpOW)AHD34|qio+MFO@fl5<5Eb)XV4A}mk zv*U*oh|`6TVS8u5&5V>g7Zo16GA*#Fa)=wX$^5b1EhM=6{cId9L+3RDP_+MRh&a%N zOGYdKYO7e7F}P{k#8n37U(6M+={jXMkwwazO?NYba;eocJu9CshH2jk&6jp1CfHLl+@TE544z=j%7^?$AkddOYE(Z(L7oP zeqpzhmZ~|b2<4cxG_*tyvV{XgMI4%n+5 zR!-c&SqJAab@oix?^My|E05&+7aUfPU!4@4Hk=HeZ2c$O(@pdIh)vG98!J{iT zlDKb6xwK|G<;$)XF_t8X*y!rpn@hS0hOZ5IF5E;T``%mbw8+ZXZV(YeR|C?|u=ty; z`pVcB^@SAsSq%$5@sc3GGXngCctXhfFed(lJZeFH(a^t(4SOf1Nr8TCakl2!%kne2 zNKJH&qmI$sf)n)r4zZ|O_KGUk`v>W;JjR& zEq}^=KGWohzi@<*sJg!f$FzrJN$-NiU1sv*^l7mYf5(M(m3@yvJY&{F|Lm9rf`bm3 zgCvad%~u4+T|r!x`6Rz4Q@f(g^{^S;pR7;*n9%|`Nl`}mdw?1rk>DTA{uelPHh8n7 zW$KWt7( zyy{p{xjWPC&F70dW14iF4V% z?weQ{`{N#ExXJQO6K)gXpHGksYm&DZZTZ_xT-B}g5d20K#v!oFbapXYE*xdqT*cwN zl!;sAwtS=FKAq`#09Ac!ecWdT$)H32di4dDsYDqB?7HH?(%wBzKiIl|N~jMh(K}Vx z$JMD6Z_e2#UQ7FW2dUkmVB|8aG}-fOfVB`eAIQr=0d1Pt7`J`}yCRc=qxvtDO#aO% zy?9YcipXd^>M#(_y?;YD5lS6UHH)Bdr`Z1L!x>0exmydrrVh)@atRf=wyw zppI2ToMVsa6Ou+`8!LnyA^B3wA>;i>m=O8zwtepkb`DGozi~0^w`rYARO0EHA>4t& z@tYcw)bXGD8;D_RH!Sd6{MTYCk-DwBso3F3spQ?3I0=gzxM`wVBVlNcJJJh<7F!b7 zIzcvR+1<`IW1rsQf^J%`&dq$%5-q)q%^Pq#s7h%^NcW5k91nJcgdr56##33BZyX_o zrnx4gZ|j@<-T!|~y>~d<`~L=9w6tc2(wgmYPH~L1+EO)Abx>8S4r;fplDV>> z;nf%0xi`-p%%2zlPEieV9V_ryDw*iS6f^-7<^(&v;IT zHAc0>8+wN?DadPo=Nk??nRzlj*q(I1Z{2W>qA`8t!t;}ll|Iq$Y11#2S+&+AxNBv+ zIf5Z8s7_>Gc<@DlDF%Zba4I;LvOoC=xSq$6PTUH3Q;bS&i*;i=x;IRNz|n6`wRKs* z!^DsOdCGaDM~gXWJI>gbPB|kM*Ni+G>0UKlW$;QwZR6~oU(A1L9-5zx`BlV8I@AO* z>J9JlP(PV51UTwyA-X49!2D-`UZR;4gR{)?m11yDf7OWt#WU z#Jwlx{Dq_JjOfNiH$VU#wc?|Onxf2fDe?-bKc{vh6TiYt9*mOo&iBoHr>>g6x03* zTP}P5>F$RQ6Q^VKX!ZQf^vc21N7q6~)&;-jmH>^k^OM@cA&#hs-=~4ozYqKoUrK9j zBff8{?wjV>^wUhxHKsPn^0xG`kWMXw9}u^Xp5Eeaig5!LFJ5!rJ$F2^Ck;Z+DzYef zSB%==4VYi7Nav$xbz8Kh_eBSViFMk7JdafJRD>UW-S|fku#Yi$@B8~w)Qz>Oe}B9) zAb3k^>2Ga3eC}Tg>dM*7_s8)bzKwagWANNS{Z&e2=uXr_jI@l(ic07?b9cr@es&az z@>1~NgksiZ?*!O~Jv}QR#>?%}e#M>}ttXWNKwjR)adhJ1>r&Q8X<{V*GVYd{b!hA? zlo++hQqDb?A@l2A7rX(?)h0|E_(1xG-*ztOJBl(-kt8=yvc-^H=^Gyw4Z>^S0o6CV zKEPpgTdcDl^riCRjm2tq_{bM&sdT=j#yWWJjXWfU(o>i78}(!B<&A)w!1Ka{+%B(D znFg27CDIkmRs^~o*gAf8lGyE^qc^?~H`Ihz=r7i>ggOj+hlsRAgwSM3i{mbi%b){X zRXb)`np8DPRO-2g+h0}bSMEVj-;*qt<^Il@+Sx7psFUjRz9_||s86+~s8`8`?WN-x zVb?L8dss+kYtz&ZB3bVNJ0<(Hf(jQGQ6KRg*tn`n4YsMajjlnPvi0OS#)t`!Xj3{4h~<&x#|?6iG{cAFNT+BKhOi#DoC|%I`6vLH z;J>8*;dq{s`#WBv;Uq8bzbz_fXXY-{qk&z#Z^TIT_&;*;E|0z1N^xqp(COR<-I&$e zBKKvt%&$gu+|iL{$-A_vH+iMFAn^*9g$re2A$&W5LMJt|<|A2D1ECh_!KsId3}vI5`5 z?q5#Y)-;4{!z8p=<5*Zpr5P+ylJVV2*>g; z+eF?j2Rn|(m$}GmT7B{|m-=F)2wQ;M-0p(Jy!Z0DrLxtXBOvp`>u zmrU>QcXe8xt@@=q?v48EJGo0r+fR9T}rKy7#yF zJ&&=BTsRX$UO4&Z<>mUhcf{v0$9cov9GWyo0-wGoQS?T>=f+eWUu;&@LoI#vU7mbW zNkNj-i1h{G=U==ep9Qq)=@Zv}^dlRBP3*^y8@)dHBiG5zc|XcmDGzHsu|Yjhk&gZ+ zia8%A_FPd16lUG|gIje#3*8u#3$P3y1__@C8|`Aip3*55%v{(tH=SD3J1wV!dDDqJ=c{*C$VDP&cWEyjK^vj~p~eC!?IS!V^S{XyM|n6gcA zfG&RI>>nex=;LRB;4|wXF{Uf722@B5 zBElUO;6bDx?2sR3WdxR--E$0t%4VJBmcnt!rh_ET7^f610n}&a>_# zyP?Z|Ha#|t`TR?om}*nuV51+_(M%`Ux`7}%j}2gKWJGs2;^UR5#~-iwMtsz*na7HU za%(u!$?C~BH+M;vAT~Z|iRkRQ%x1o$SIE20Jao}!AE4VYhVB6H4fJa(2InhN=d(_` zsZdgsk-PVYt275{0qbHYx28Oj}}TbK2J5i9&6xvAzf}$nmeCld3Tkdob;55C7Z(i31Ksl>dw+cHB zU>RLPBIaGg=f2m;bAyt{PaybcM*|@pe9k7VpfREYheU0`aZz4TEPC@cep)ogK4%9& zKx`iM{b^`ZB#NOfA3ROx669pkbVs%e($sx;2J8BT|@Pp9w6J3kOTB>Wco_&3JH#ex6J&Q;^_KIK&= zgI%{w_C<9s+`945IIrr=Z@R!*BHsI=|7;&NyZG`%DOfQ<6A9|gC1p+x{sAIO>)B*` zx2WA#zc;rOV?ls7|pb~1pHcU&CGQr6k{TR++u24tK6{=bQ^bEfV9 zrCxSzpYl#UkDDN&Ce>M7)c~C+c;1(-1ND$1zH8vqAMSD@?GhLmsA1UwKgrl#Y=Aw0 zZ0%W**l5*ro$YcjT;{njXMkGL?YDr_1)G`isP=@I&h1%rHuvU8^^3gRe_CI*Bl`JW zFAs_8%n+2;xl^4@1=dV%6YJ1xTXT22&{PDRqx<(IF+Zk7_U*$$lk{RV>pDqRUcXKT zM$p2Y7%x+cbFTX&6W;Z%n&z2+;{jk2fTVwLov|%#boer%U#-}+xhxB_Owr($4ddle zq+ODGGNSX=R<8cFCW?kf#K-Fe=V?moz+6`Dup6vY$(nfO*`}4AI!Sc2B^;KMNP6HK zm8kU|aypnbKSfWIcXcMP3e0eAL_K@--?cqSPhH)Ap-_Lddh1Yt)>x6a5{n4E;y=c6 zbf*Cfm(!9)Q;Zo_vK_H*`uyA@i+}}VN&B0dukYh_bs5Uz?%B!s#-tO z*04KOQH~SXRj6g@8oaFXV$A1%tum~v`ZfCSEiiNMk^WR=qp?l9kiRJIZI3G>r;Joa zl2l@tE4z0DbqoW<9R+X1sJXtGgq2XAG6d#5{ES;Uy<5uR8n zKG_g#;E1x6dGUm8Q8k={g>)p(#0gxDpJqYbwm;NLei~lNE=3}K5Yh9VGdt%iHkgZ} zxb8{?WT#qAyM=mF&1!*8#@7CIPjuVn_6rDm8k>&XpX|ruU9YG@?)i)FwsF5eO7Ml* zMQM7jTX52KMRnCUb=E)Hk7nNmO>`yTv&iy%sv4`^9EqR@@VFE)Z|I%fsxpE3dsny}3h{Y~AVLK-=-d~o$ zsYuy?o6boH3+B7Z@yLP3Y$1z-eqkk8c#Bi zh@G7eLIn5NOKvtjHS}Jj=F}U@S*dp>5_DWcw7rw=BfCaHcIop z&4BN7O=2K^aS>eBzxLS&2d|Yqry%;JsHLhHoVyw3Zwt&n=GvV2 zQBn!sF2(Q{MIG?7po}X~4qWO&=J=7{KbzrR*;o82-xTJe9lDi{*GazFsJjl$bt~$X z(j2fFE53d%V`|e*XH9%palC)SNO{+p?s8}+=6A94yTaX;=4oXE;%G4T za8Jc7U0rsE&!G3!&d~j(4xy!D?nrK{_q}t%lxrjHLQFqOXW+_AjJbQ zAQP9SW>r+Ar^AHO5@1(St=bp~P`{NZfFj}kBe(iSnMpRnt>9K2@swjk=55#^yA7r-twFgm|9{=Ew+dxfdCPj@ilx9vxP@}p zn=6km6dsYJIl3pedXQ(JbG$dn5ckIRZxk=DnS8ClD<;y0^2&87UR#UCUgH;FRs)NB ziYISU0CVRHCtoRul)TqHu5@98M?mJXl0bDw(pX=n7GzFz1r8@RPFs_wOpY(KQ!AfR z^aDDITN6~oqTbWS4qaqP4bEzGgsudSI>yg#aaSzpX3R?oYC z-0xLb(|;%l<;FQbzvRzVHtOd?_2gfreIfIA14o$Bj!IcHS`zG7X&aBII+Y}-+=r6wd>mrIp+nkIDsa=6wGcYB z{MGWc4bv5_DxkB`yFH!3fF2$oQ?PzaHU#syGFszsGaX%pS{^CDN1I_EELS-Fz3kjv zOF<5P(tzd_Rs#AtX9rQt*EW(F{#k;gh{a2=-|SG(2pR0)$|{DaiNSX^$-IUl{K!GO zMRyzgPf7h2)ZFGFjxB_7OgbPKk@BR|()qA;)jwH;DPwIgmcl)(69MMnk+f&ka|e>C z1sG@`j0OdP6w94l2|>75o?|sRr=a_G%X*horvzJMRfT>#Q+-L@K+hfMQ-KVbS6&5J zxo&g+XNSzK*$j;%~_1Get9we(y?7iO}|{HN!+wx zix(?m|LBO7oI*YF zL4FC`;z|!K{G;|toy8MUsF)x0rKXy$mO{KgFKP zt)MrOb!+Z@{PyE8DkyBkeZNp+8=B*Q;BNBaW=Of9p4r-ZnTX-98e}JcmrZna**oaV z2)b2JB0r!+so3^+a7tp1u}FsC=4Kf!7*fC;X z;jDe*iwldF=SPKR{`0C^RTm#lj27QXpjBz^@(GMj^ft6)4UaYLvqW{rqL4N*6{lYw zzx4V9{bHeyn0x#{zHa+VY?Z?I?k2zEwr^iocCpnSWn5%nAd^r!6=m|a!oXwc+>J)P5|#>ttDEwa@|Yuu#&S@ z$sc&GX&o7OLW!n-^*gZ2V#?#Z)dVuEf0k59V|ubyx_oipeEZZCotuh&Vh2oV7#lZh z(dH*UK*(Q-bDdFIlT=ggam9Si3S+GW%?wqhK(j*rhK=E+jUvV_^~bF)3Fyqt>Fnys zTO$@-kbgfy9@>x!*T>Tq{hLZ;_BQG~spcy9EGN5kK8R1suCY6KvxS^`=DO!@}{JiF`U zXm|9lpF?hmMf3JOP*2Vf@Ua)oFbp1+41bJjJvRsYaYA@Bygc&Udl=uHxPGJ6J#Fc6 zDEm0ELt1YBvxNY$^{T4|b516i^&=Vf1a_1{_@C+Jz zoc=>v9VBEIh5o6}rL<9R%0#l_gEl|o@uws~pq?yjBZ~WOlw`OrWwhTx17Wv|#?!te zdyB7%6k{xd~BvhYiwKC5h_c zyhfPvqYd*_jg52Tar+aeXo5#ScD}k_iKIlrjg>L0MVb8lu%q7TYh-XCpSC5t_4vuLys{?+Gyf zJQ59;!z_R-2CaE2yTBW@F*j`%nhJ~kbr-UREKWhssY)WL2g0Ijr4|I6W!Sf^b2SOH z?DtDG3d#MRakFh9yFArjGV9}rS5s_$j+$~8+{15H>X7^1$ggcZPX4^rQrFf`MHR3a z@Ue6!D#ufM{RJL}s`E;plmy8Utg#^u+LVRQq)j1-bQ^V72Ld+C;UnX%SQ?tM^NR(c z`{kb0ni^$-W^oz^QzXy6 zhCUSm!~qJ(1cfw9cdU<*HnTgG#DwI!^)-6BXzzXs#!Dx=S?z@*uJ-AOKxA$>AjVSi zHKOPv8rQmhaBT29h`6YQj-|xS5xWVsi1(MzROgf_m`?#?shx@Yd0GQ@6C3y99t9qD zmt2VuK0DL!mi%0pajwO1Y-yhBw0OEdRgd>{sh)l4#rV+M;^9AaZ?_4#Zay4}qWG5#K3r4^S9E(JXv}l_OhXipUs8Kt>m2;A z=)818`s4gg^+*mhZLZyRAedHjVBTa0ZnyTyZ{6fS=C|zpd83f7&&biFwpptlqstyShN&iw4@UeK8vwm22!XtS*Em1C7tH%?2vmq*Q zSH`oE>^>)Lx_j}>UsXd4>cz$Ruljzjz#*HC>l>XI6c6(&ot`q*i)+xFBK;p63k2vJ za&FAGvTi)RvFgoMe7(GZ8&+{e+Tm&;*~Rv+ap594G2G5!ArTY4&mB73P{`U!rRn?P zZRbtlNOR7-A~R2S_A2jmem8bR2vB+-`@l10OJF8uN3k6oU#zjaGG$rkqGkUkpV8vs zF4j|<08h8Z7O%M3>%{u;@b0TQSFD)Fc6m0eP5gL6Z9)CRwDU0$s?N$3?mc&ILkrHF z8wO|1HB>6qj3YdrdW@^7c~v;s_Bls{O>&t9xM0_vmJS@uMywbswss9YpDIrNXfB<@ zETMO50TYp3kK%L%^&1UY`{fC^!NlJXo4bkc0Pt^SPK@5W-&=}9Ei>BJ?b6OvI@wsD zpTM+N`oRwnj17oVi~iOXK82^V+u6nDYQvca=MCTEclg7$itp`lH@nO1C!oi71mVD( z{L<-x)Z;wQN@J!UH-{plx5zs#ufat0DaTJUW6^mrK0-%J-eShrpV>SbmbjLBuabzD8Or^q#QNWjZ>DJ@wqCeK-Js52a*zL6sngTf{fkc^Dy%#aB zDhkKvCiC=`-KWOy4oB;S7N)jZfL&`Km-J0KnLQKuJEFU`eSuo7p_J<4* zy3>u5!R`$048yz;&$@w6C!SC{4MH_F=D*VEG9XOQKeEXaoLk#VKVm|_ZtQYblRt^X zrqNBAwFAw<;9c`o2!M<*O36jIEHIvk8pAb?fR~a4pTpF{G+BK!7FT( z0T2odWS*UV1ky8l;MydJqcg5PzY^cVE)4Be-dQky`6gYLR4yAfcC4NMqM2jpktpK{ zoc~d%S>SQwFO^|u+JCz|wgK2wl^g(YOMx*wE(%y*x@-MEd=-9iM>3Ii2W&g~I)rHsWFY_AP6?%{1cOcUrO1$dgNFVr3zB^J-0a81|3D zreiCShwC0v7-b6cTkCNddss1v++^7fATnbGRW<|b*vq~7!G!hk@FPc=whJbEt+dvp zmijockRh@N1B~^JVij|lwk(1RRuzdDP9^aHm&XIcB$G(UeMQosrJwk!X~&_yxyweC zCFHWNeR*sv*OjXfObPoPtEvU8kSCQ>{oteA~ z8waWMI5$*h!M0;bh1cEGAYjrN6*Uz3(n!B1hIAiG}9*R6+VlB4QVO3CN zxe91Z1b_!A9-2`9|3=}qcl~-IBE0Rd$<)PqI|cuzProJc2Nt%yJRwd0H}9`?R6f1N0q)|jJSUr`iK zko3SsomS^D<*&PnuB;v3YrmJf_*5*2W|&oF%8;g@>rB3gIPmfCdmNnkBtSWgGW^W38Dx=7`}wlLKciJzI;@mm3V#<~f>Bk^Z;cvdyVyY3<DD(Al(w zllq}_B6ZRodQ#q!msRY#z^vTOGVv(ZL%vGyUv$10iNk)mq3l|6HjFI>?y~pQ7rCzI zik;ubFCT)qi$pweHq?Z*wB|_~J>WRN=-bGedpTF0T=<~vXZ&$>I67a19guG9fTY^s zuZo2`!4A^Fj-o9=MgaP@R#WL@yVnG`7i})rd2*$JsdmveE?O(ce)$?hsKUwG0?m3B zv^?sI%d$jOqBfHm>W0Zu`rtT?K2j7vgTWs*D&W62fD+<=k)W&{;$OmbYLMk|5o~$o z_x3^Osg#&|eQ&LLHU$+2rM2|aW&{X2o4s|$ z3!l2Ehf!bHZBQa%3vgxUi4sc=d?>92g^`>kH?xAMEGYJ1Hu0{SJ;L)#f(s<@7%?x{ zE4{nz3tfV}&ZhcH5|N)( z6at!crqph@=eGA>?bTt$H1mpSg;t_}E{Wi}1->g>CrbM`Ls|-8P+BF<+#HPF+kmLN zF9MK}6yu9In!y3zr$>B-<5EN8x6jBqLaq|L7Xh7QnI;3T?Z=crtSxDj|Cna1*klrK zxyw^3wE8dL;FVRhrz`g-vJ0OCK6r3x^Fa&c`sqr5d#7i7TXLf$TKQ4jnVOK;2Tju_ zp}`kjvb_#qd2IF9yZ*Yckvpi05bwZ>CP>eTwVlwcLTRI5L-5a?QAP6?F%Iv=C){{ZpU^vBR0Q(#UJ$i0s%GF*Jm69UVl=FI`RJe}TVqovM zx3AGdcM_=MZ4g%NFpPsV{$8qLm6I>KG^k!9i{f%Q{Z;vJU4uv~S7GthGaeqV+t-`B zLpP~=I+?HUIzRzp?1Y}2&Ya@#i=uR&tB+Dm=cC0kUF9(2v@~@8jryMsEK(3VQ8KLq zC?g)ZHP!yaJo3~ovF;OV4D-~kYU%@e#*fI^wIE8JFaG1W>w~0yKnyYda~ez8-%(6A zwTYYMHlZyuuQmPraP#jXO`J9`bQw`zp*D2`w8P}{CySok2&sU+BJu$3qN;iYhySHf z?`?$(|0_W1FEx9cNPi&Sxf>!)fT({8zr@{#5fYyln$CP1NHq}&Y^JZr3n_Gc$u*?U z%-rx>iWccYN3J`&d7AHEjlb3i? zE@Ids&RBpc+bmA7AAU4`VMi}7yxZGpIC5NFE<^V{B&V1?e{<&QEh-J;HLSeJe2UUD z)1^^;=*&?)MjF>D7oHiJ0dH(*)0+b99#OSiYD;FMwD5SkyGNrw-E`8&X|9PGeRHNJ z9|6_E(q^sVcr}}rL8~wL5GmnJ<$6i zb9Ec!`bUyy4*<-n9o*JliYIxt*{a~$#>@T;u5`7QxE2os)Jo#J%af<m%UO+VYL$<)1~ykpAGwXvw(8V6JSy_0iu}C+3ljz(BX|GNTYlt9y5=4-%ibB z0NnA5z!PZN$Nxr5x0*1&O&qH|@%_#n)UST-+zFT30{-p?S7%zcH(pgV?=nMIR2J{L zE6&`g`kt>Jepk3{9(3mHb@?6H&xR?Rtv@FRmkk$=dZ2$zbPWakA_lteY)wZhVSzm} z{R&%Mov*4_|Ae^Kgk~&dYE&Df6XzPpR9dqcBC`sCM!h&iazqzy-OF$$1x}YYr0PI? zao!c{5NyHB@W?Txp)WSL>7zNA22C2(8z++#3_n92GjO6h<<3$0mEl$m%M87>u{nEl zw^r1{IF)NA4w5(=Z2O+iA)oZj(O7A9rf? z@b3xx&Ga;YEB@03Y5`zxiwACXwG=q%T~O;NUci5|M*3$LWq9DnovL}-wtEC0w^+Z_ddSN>XGn=+gR50G^emZo!|u)Q8~=gE6jxcQ z*<9Myq38$;J6eIk>t!|kX_tgRf^Z3n_elsmzm)wSzw_ypWhtHlSuc2A!ro3J=rHku z0+WAx7pQceCQrB|16-5~ay*=Mf@$Yb?TvDgagThe^h&%7#&!WIS(UuGiBZU0XE0_A@k6$K{I~O&)iiy zV0r_p89>y9r0nWssKg#t=ty}2#3%OheGDh*1rhSoZ89YJ5~84r=o5EHEn*vAYP5~? zjBm-e_>#UuZ)$;wF=j&hk1#YD*vI0`xtt=sj`;fU#Ih^5xyMp=r!#Lh%8#^s>6q=c z)feY-54d-m)?P^Ca#&{Q9*V0RqZy5p&kcR7O>h5$v{`DG3haaWdk_6j;3U?1gz<~O zZyAi7QQ)Y5zwB|4ldqJ`5?$Gqjq zpwphL1>3(z7F-|}W$Do5F#bwfo@Q7~+icnM7Y`k~FO&irtLoPxG^@L-R%R*WA^ul&=* z6%8QGK}M5YdY2~_u-#}1ncG9GjH=+jX*pY2pg7OyoON|o6Se)@u;EkN_{Cg?v?tj_ z7xnYQ_25lL#ML4#m?lm$9ZD>pHf5mqFK3XY#rHO=lxUZk2+p|p-m1BN*akpnjJw6J z`8^m3pZm+GMd*^Zgm$eWBrzfyA{H{54}4N$>LKNaa$>3)nv;WD8+pUw=iys#WUU>< zs_~pSBr`6C+XOxNwqE02$gW`&G4>`xvUv!AoC|uqS3dn=AWnIEc}f&6`Og3MSoC38 zc}a(?2=#Hw77&K1n8SMh2LDc%UYes%!?eEhUU_1WjIatNtg)a9^4EJvi>ly(i*(9f zi{>2l5>?M!n(8-q)2dnJbx{>uAf}P8qdhL)T} zJ^SprsH_0iTJ`2TPfw0f0{$63!?&PHKHQ<8OzS}aYbdDmvl$*<>FU&n+Ghu`>6K|( zvd{;x1v9q?m&|3CjclY`m$o&Cx#M|d4c02_11c0Tjgc6kvKyD;pq%wh>|p~}^W=Ti zzi)s10n0ssGI)TesZx6lj<OiTjC-Y}#Onr064!zu|$f`s5?;}3)xI4tKd=&)n`Fn;8zT4te3pUr^ zKuu4ij2%X$ZmY-2YW~?TO^rLd;_k_)co8=f+;drpCcc}fzLluaw@a|tlTF##Z4mW{ zX-8!4Ywfvvr%SB zXOA?b!2u_=S5tskfa2$#$OyNC#9UCp^c3Hj-gKa_sU9{SFfjBubU%v}Y$t48( zvgFqxYRH399o$*&jvL+w5Yxs;BJ*XSfT6+&sTB9j*dQ0WI;7Fkh5}SJ9Tko)%bJ1* z>;yn_Tj}6r(Gm{RUg`(P2{kkwyuamS&Ap7fAN|9J22wQs67Urd^`~OYbH{Y0l}H3g z>m?(-bIT42p8y2v$;j=ybFianvtiTaGH}W_(l2U|7N<9}OZuOoORCqeJT*^<91b4u zG;i)T!U4TgPmHw7I<{vo?pxlbm16@DZ1U;&yj zRje6TGB^rozUmc!(*lB5s+)(+%reIMr2z4RVF8mMkoYdLXG46?c-6lOaaj4{EQx`CbjT8zXYZlyVbJCy>H{^O8p zyoGJ~;q`Pm=QP?N-wqTvby#hULJp2BrYE*@@s~D0G@u@$DK}^~mbYyrE+D*u(TnQ+ zRP}}{m#<-I&3HH5A-u_pkBVUsrLHkVfWva`BmzcuA$gg*u{{dPxcx2uAmp^tI=Jz` z4S$Tg(?E(DceWA*bF+dT6coy%-0h?M3rhN%A{`tydq-o4fKy9t7ONbO@A1c!Wue%< zq{O&>mE?%NtGJyw+LY3&NdFUN@5l?JsClO5#Hr%AuFq)wdA*6k5&@6RS>W1zvldeH zThhT~LkWPq`ujza6LH_JMu|ER{@9=K4+gY$!*68k(;P%}Z3n0~!#6+q9x^5{-)}S; zUkSauN(OUTbQD<+(yF4gt~1mZG0?^31%&evob6mU!U2Jirc%-Pi9HK6ovn*cgBn)3 zNUcK%E$LbPZ#q)VLk(95;eNIFuYlK@Q$ZtA(Rswc_c>l<9FU(fAbyo4CN{-Ysy_x*M{| zmFk+=<)-M+_PMxnDEsRx@atX1HNl-(x;$ZHFWrw*v7wHI6qi(jz<%K)*lXjt*FPS< zem9FcaIgU}i6RJs(i|AOZr{bYoACaF_pEXA-9&^n+;kH8PcFsvdryD*B z#EZ?y!35X8^$#Zd)3DqFti0;Oc(__#tz@HD!3mIQ%V#Sfb^OXej$FNZ`PV!OzKGxx zSk`hTo-$d3FOWZ6N%YDcVe>YJ_x<(P+VruFS3}fhK;@o7)@L-40R6Y2_TuZSxF&Lr zlR7|irf)!y@TX4xO(Y(}wacfQhawp*L|LuoA%Rfs*9VyiBbDl8&XShIJ1FjP!R&vR z@MqiLGP(YiyfjcllQY4vx_cNELmRPg$!i#XQDv>pBi4!4mT$}Qhx0kV;Y5FXZT};nL zvVDRN(|IslOYrnd8fO@rLOa~?A_;J3=8%WMtaa}0FrC5O*`~k89@0q5G)~n_2r%Z2 zfn4wR=Wvp7pT~(0(wqa0Se>!Hc%OB<)M1cbWmd$GuR)!$W_SbNe)SsZ*uz$1_uOG; zNoQ28OxR?6^gB!dKn4PUX#6W*iBv-ldy!-WK^RW%eoH2NhZk99Ych?qh1hdC!6oAB z=H3hAxSKnI2Zqvq?#HTIjywnCmvhHF0VJ68OOM=omJSeDRFQ|{NWFSmYcZDVrAQ1zR|i+xQkVcPeF< zQne@zYf#AR-8MGNws1QCryT3MNt%h?nl&HgTaTQRaCh?Dm1NkxUni?QoZ=zh(zSxC zG}gg%+y<82=u!2~%%SKVA)^>>>fDIF3yR<0bneR{Dz={)I&WkdvQA*Wx-GJN4GZZT))XZ7@A&ix|r z+6hu%KP#Ia-+(Uu2_wegR>`W$#cGbO7TQoFZ4!kO+qnNO zsM9U72t~=JDJ4hs1>=?+#FyMTS4K9^#Npa1x6a!W%MG^)q3^5FvFtkg33+E`ooUTH zEd6BtdPR#Ur!*~KYe^B@qK|A%=+JV8F^rvW#Z(KNRC`u5tH zFPe~&!DU$;-uuAbktC;IGaR5yo&BKekxWk^Z`cyT224rMYx>j`5#bKn`>+2XO^8(} zndCO1WVZ8{ISN3el8+0kc>cw9WThb)wto}=BrZod@rN# zP5C5OA0YS9)#Lg?IEonJtIXbHWm*R&+(Mh8l7O2hKCnw-cGIgj%@P2`kqf|idu-FK zC6W4GQteL~7;3FeJ(K%q<^CcV5Dpe=3eq>R9wPKyRho!&>eWn*9QJd%lK|Uck^urc zh&o02Yctthq&Gt8sU7OB)}l?`Fzv>+{NBU)^fJ{R=i2J5h^;kxenOv0apVS`v=Jxg zm~+@pR2dFi0L%&R<%}q;nq!9JUT6X+VtjCi7C?q+ij%pa6-D0+9cECRzQ+LWkZrM! zcUCjRk5*S4{K8yTwgzsc1T*H(cx)xTdL1@9y>jXBQ6nnd_~J}cequ{XvHrRd zNevkm?hsBV<9y_u>9lsCRTft+W0zUeg8A7T}ER_v>d*)Tef!W&^2^fh4n2^4(FZ zY-47^H^GjUR=SV)YQFKLU&@fF4X_(@$ij2E!cwI&;*iIa^HyGs<|n?(KmhD+mO|nX zdp?ykVwq)c!C`(4T2Qo0JtLLH{<1t-oObjC~FrMf4S?8(Zg2DfNd9 zZy+r3qT#_&BUf=(_ab~@{jcPKCxCF!=9sDA+ri*snZqtm(g%>1AhMuQ{&4p!X(Bi+ z-A1BP-AmQUi2&WB2Mv)ijvOFZ;~?mrQO~u$WT_=R;1C-o4&7|}!nox#o@S!|iLy_Z zOxHruQmO-bfiTlmihrA423U8fst4Nv$~~usYSkMMQ|*wAp`axe9zRG{Q|T#pup>a* z0B40Bu%j!x2l8hNnDfeW|8+M0wenTG`f!-~a0W{G`{hu zPfy&=aQHwfAdi=F*?x5G$;OUey-4#u!fY@V2^h9~-z;!{UU@h)6%})v{4iVRD&rab#^_Q$Rhz+&)WMs=?eR1P**7`CxbHz=9P8>Dc=@SMe!6iS7DL)=F`d2}_WT+;%7Z zr#&YSHat%Q2_P3JK|5sW&un}t{uHvQR%TR*+H)PZAL!;Uz5-HLR*VdV0b02w5a>_& zn{Wq*fc=l8W0k*B`kB zLA(Y7)#ZsOC<8qR_1FG4<`aSmGs6e^X6|KUni;VK>)rT#jdVj^)flXYY_g zQyYBQCCJ?aE1Tt|AH!0@aG_d*QV~nger@IcA*!^(Eg% zxPkqQRHpPV%RAtTsP5Flw+EKtfL1dqS?+rCauijimu;9%Vr z!Yb<)7jrePp$hb4KM|CM-Yc)Ym$Q3BWslIah!IgRGN(B0J-Dhdi}fP`2M5qBs*rgBa*H>o&=Sf6DE1qqSV;2DS9o6HRc#oV-im?GPG+Zl6gAsC< z_JTHAOmmgnlLo26WgDwhnM;6_1?y}rfpGWK-T|ej7GME3EQj?+Vmr|Hj3`ln>}{*P zIJ1od9wDN8+&(QY?Y@O)#!*xhP*gxlL=UjTmr-rtxV%8APi_0< z)-*M?m@hte`CU0mkfPO3bRh_;9hyYuv8L_Sq6pu=lWP>ic8D^Pf+O$PP%Jh_Hf4+8V@P_%QcsvasEDkP3?q#upLIFMRPi5NPCi zpfuPp!)<^?8FWT_t?FyJi&US3lpMaD}gk$9g9$KS&mZ7p#9wo=s z8mtigk+;8%caykU=G_m1wq_v-8(T_x{lKYl)|;O?OSq+Xo=ZBvi58-U zmy1e{`TUyxb1njyZLb3KQOBSUv$YEyo4BXyBxS(cdg=_F)pazC<~j z1|_C{Uj~oNct8I_58p@1V*b)A z6r6oLd|uIiSbEe)j+-5CGvG=r)fIfXd6{8T^lM?eFgyrX`2f+gG> zY6muP=G5yKu35q8^#|2sibh6m6SbLw$32Nf)r#JG1k>AC; zt>{WE%xd61gj(9Qp|j$vWd}=W+U$h(HMUaJxfUe?3Ei%R*3{s|2!@#($~UMlAdQsD zG+wHHJr|E~e@u;^HwX|{S_Dia2-nM9oR<3X^*bfpd``3!F)eHJ??GjAv)*}D+RV{(*9|g%0 zy6oA;#*^R<2M=FmmQgM!Sm(Q=O1qjPi>lUjfTEUAP<-X0P~Tuy z((^rGVZD@Y$2?z+!K|t+NMZ=Hq@Qfcl-53IwprIqkJ@z(Kh0WtdyHJXIW^HTM^1cU zlb<~Nsc`iOt12Zp^WIvi()wHaA=Q0xj<_7_=o@z%M)-Xb{QS&v%YRP^xEQ^D;l*s0 zx3YJhy>Cx>#E>8s)BD%obEts8k86o|y0^J{49WU$hY8j}_21ZuMeyzUdvWeya zJug{JWkT6K7-yr~-Bf?=vQutzrPBI-NYv@$;B%PEPJ{@Df{322{12{Le#xsJ`YOl* z+HmZQkJ*Dr>VRBkG6p8g*!cQPuj8{jS(#Z4+s`D22NxhEolrsfWke5>cRXHF1!c`X zhb%naL`a<*sT=p_6Dl){P;RkYb`rHqA;z&cqXpM{WqSPsG_liNRH$MB^c~{kxS-mOM zcuPIM<#)){vXI~wr8TT1@dDX)XipUEp8XK!W>tM?h7xkI#B|SEeCFopW!&kru4VMu ztQB48Wk^#4PSGDUxOcr!@rx#T^JZu1e+t6Q+L$ktoGlU zhrN@pvs$)Ap)gyu+<0{OMb0~R8uKU^>mEB>Po<8s{M6iU9LGOVp{cnCk9x_lP6a5W zQ|?=HiVoXYWq%v#kZyMoDBCDkh2`9$Z7q1~&UjuOgI?8a@c1lm*05srLaD>{fP4zq z`1ZM7W>b-TG|cjFMp1M~!JY44AVQ1R0fW0=GYi-|-DEwQh@te?OuRZT*_e#B*XB6} zOgM)JFq|7ZUUXHZ>-0cP{V+@80iTr49dt879UAx;m1g}hD&1OV=Q_LI>&|Nec%pwRV*TM(NQ=_OgK&7-3Y>f@gM95u>$(mo z-I~Dj>b%;?OcO?0S{e)vHXei)p~&)bDBf{CNvopJ+JNFJ40FnjK|AHXJE7|WJD?mq zu_yb+q;fX6$w0TqPfHo>IcT{bO&32S?q^O7LF#ARc8!T`)XBu0A>S97|<1quE1$$gFtb*eBkmo zQ;I{W9)(~n9aVidoaT5{8jRAG9xc8n-Ct0;wD7iTX@t25&+JWC+aJ7t*$TUss`@qp zQP6lDQSkPFJZe4Mveq6R}5$(u{JIlOx&#$=tA`98&FYQFUfx>wG<;jXw}L z)?fxfUDaN2XHnwk(*hXXM-yIv(;VtOI^w-*EX(~G%9I3Vh@o6ZMs9X*CXy`6hD6U5 zC2T)DYv*VkN&_|4XH+WU95^zepraxKYV1;k#o$|G+zKKm>!53S&l7U1Q|rS#_XpFP ze(Pxsb11+}wn15WD9CL2QOIl}5R~zw{1kAkp+a=W+2`E}w-1z9-u$GM@PcMA8-!SF~q>f84iy4C}g$;`-ZX@m>wbbirPG-Bpg` z9}N4b!z|bn(4{v#&Zsl2J}o2nwH@o2IFGJbN9&c9$_e7+V`y%!bu{OR6{P910iV=H zU-{6iwM^SQ1ZxICZ~wka^-#C5q$TJkas(M;f)uaBl`p^jZ@6D=h?e%15#A_zTS$B2 zgNFrbhiOhDzHIvCtCHb{%i4Ju(c%@Z;J31+ApWc=Mq5F&UZ!d>iw0z&x?Iahgi~(a z3EeNATFN&nlT#9&y5?OLYv91SQ3!2zJv>v{rFd^r>}p!=I8E*x`Ch`W>zY2xng=Dq z%ws6d)qc>(Xb7Em8J?zyo!Nk5#rq+`SGJUDAEgSv)21#4f(pO-V?88)R6)3;4k}1| z4p|=CgnXww6zmRXOzW$3=qXrO`|*X1)UO|O%wdm^vGiIY$qvced@+$V8L-i%m`WFo zH8R@ejnYHpAGg0STVdjxR~ThxjntR&EM&4!Eu7Upx*{I}&TYLO;?odu-igG2ywT}U z(Q%d+gX7fkelC=a7V*jz=ZzbDhBe}Xqdnu^@@kP&*ifZ)Eadk4DTSDek-gNpZL46_ zyY0brxc+}r%Kts6E4_!cP8O9B^J@fR5U;3kOf0x2_4O_vsAy+z=KMw2cen}Hy8I6a z?;0z91Vv6eL(uD*kjRBsxt2kf@x=|t%m1Ng-=OVbR#c}V1|GMMgoFq5Z(RvwaBRzZ zn~zzqbtp4Bs-r_!*{$|QiiCM!!tC#v$E2bTl3XjqdgHSla}be3i4Q$MEmQZP&aroz zfzL_sU99$bkDiqJs(cV@vgTM1In3Xw72)92gvH`XtBD&!`mFZXbSZ7l_>uoJCAzIt z6lv|_vAh6VIC1+nK6*2h@dYG{jSc6NLCIOG`UX7wjxG;sNaL5Zfmx$uch;6=YttT* zWL=eb?W~h`tz~)l6P4iZ))TbN+TML_)#9<12CtWPivGNNXsl+P-zStDTbncaD`t{j z#cm_N?NaRCO#Epz_LQG&O`P8tEX0OBJ<74L%CnGeo^{{+A9Q1Q)lzn|fAU95g$r)0 zXLnUo$;dWoH(9>kaokSa%Cve>UvC{S?9o90#+*Vo%NzsRnotnTSZD4DgqKr6tGTX| z(GgWybz$@VOS5R>H<6h-DWzWrV&!zc(Yu zLw*-=@0@An6Jhwko}&GM;MP;RK9{%4P8l#JUO!k1Ld@=$XI2`_Qc%M^0PqQOI(`Hb z^P&3=u756s|4uRXa3YV03)UmDtd{@4!m7Y)bE=w&3h9%yCCY~IMJ+^FJ{~a9_8c+W zx)ssCxWbrLaD!=7^mwZVANSm+qhgrRx1vIcAD^YHGE9X+UV9Q5<;iK0bva(OYClD{ z2TkBSwdb&k89yiF3R+A3I=`xXp8TUt`uDVgMTE2wo%Fl$ixW<(s(N+B!fwhoj`A_v=b)zNNgZoc1pI(zA^=Hvp8^ zhh^*-KAwW`0H^mQlx4OCpzy7Cf(m@ASW%xu3@Gmv3bF~L<a;q}qPqdhf%Wb&}*w|mL`ehg>QTs9Al9|Tq?JRlCyDq+-OFrVF zA1`?{&Z5jWeo1lYdQ9bX9Ob1|;lUc{JFOf;^6=ye{9IhgmM zF;TfIp2D9jXvv%<+j2qbqyQHlJ7r95SIe$Sdudvkx>9}hVChabo9b(W-wcKURF4>% zab&i>cGNPhJQk02D}eAjq2l-z#uvE7k!=c5Vq#g>)YORRr^r5qqhtweFI*_jVT>zR zw9nbL_vTyJ^oVH_(3SPAh}nXNJrfCtv5E7V1wne44<)Lm@UsBwPe*V@9Dv}!09?p8 zq3q}vZOBtzO^?c~Tu#)282!$6=)ovfqTvkkNq$#N7QC+_6t0sJDg}$2I-P}-q9wK- z$6;vrJo$d4s8Nj;=(?9C8u3IPkFa5%b^QM-jQ}x{d{oVD#ESRx!_!Z*CW??Hd)Rv* ze}C!m^3RU6QbC`&3tBO3o%HMK9eJ9S4+cjn9Cs^mV8 z<0%hv3^P^$^Xf&>GL*I40k9Am#zsII^)s(t`QF@@GpR+lzICUSkq*kh#tHQ2RZfoL za#W!a=T_CsE2ryqd-o711`3;JV#6oORKwW?xhHUl0!&u{^tIh!m!x$Du|Dw}vA&qb ztmJ*kbPGT5_yho>1Yw!XiXMENjW|WM0bm0RZ~c-`6SWYB7qnJz=PVe7@tR^+Ar>R= z1zxIzqaH7T`^>SSHE(vHo!j!M=QR)L^ER;`>idAMN9&8GO#Jj^?0+bOABe9AXEGng z{y3CDuAdr>Tr}g-nt6?hHOQ#(J*TGkXrI53MAaf_?gPch_8_JkgWH={;F+TX9+d77 zgMiNIqmaO{SGj+D60W{{Egu5Ya)J>q3 zB(^Bcvk!DMy=Har1Hq|w*63(XaHKxoTYAVQ;yC z`RlznLzgEQuWp~;FUsB|JXLv!?GZd~Z*DY9TbVr$9)6Ap(`J|MOeu!KL!6p}{G15< zA?R{}8)Ax0pUA!f;w_K=4B8vFb6(5mM2!;X=c&@zqus_l5i0}bOhIFj8FMNpX^^W; zQFz_n_rtW%`$f~fTAF*wDe*-0UM;!aYm=;vtELM}hW_;V+OVYm{i`yXe-3zjq>ne) zw-}>CKzDTc#axV)4g^zPJTRp+IHZ|h#|X;^0b}hJ@>g=`QA0V@X@`XA4DfaASX+o}c!wR0NZ5L$_GJcDz{iaY8PW>cO%(Q|nk45*|leMVDk!K5R*52ht z^eyM!_Sc@T&uJfY^q`bEKJI9@9ijTP9-{cYJ5lcQ_V^87!bx49i1668xdG-WILdXG z>gmjWw0jeM`6SY<-U`l&yFwDZ@jY}HFVOQ|79M0vk7TAH32%d(5FV`11mm}1|ZJI0L_qWKn#k07YIizSzl zd(t1#T@+>CT~PPRuD`;$wUlN4SwU3SL!B*m^4nK`!zg3L9;N~lQJ(Pg;%F)oasE36 zKFIS^St-N%YEorpGI1T{+qzj;oDt|!xR-f@w%6DQwnrK479o;_4hXX|0WtVFo3Y0Gq(~b5qV>o2Qt)t`<<4f>Epqgvc5yOT3wH(I zIw#CUi2%Nv$$$!kww-XJUy0Yc4Z8v+-gE`VYICwljYo_;n@*NbUP%^CKj9bk;!Z_P z%7Ts>>h0dCuQ{i4Cm~({CW}-a?k)^YvGwaZwZp>FZY+Ph=)K=(^S|ZWte>{El!?`r zst4xNYmmvfW`*D1^;0DeYZ?JxJZRzP{(qT_|AJ!HFTU|ar8N>JDqTBH0pCd6>_YVO zb0{Nj;Q`{g^N`3kptA@I`ta%w*}NPN^ZO)R&b7pzb$ZfpsNB_CiAkX;RTQ1Hq6yz+ zQors28^vH9dTUongncK2spAT3T9AmjF6LC6&7LGJYa@zlv9h~SUAAAcaBIR-Sd52L z>vWH*ce1D{?Evzka3hVR;%>BJEpkS*nYlCLiY-mnWBbk?HEf@7Th{KYCa(^wZV~bh zh{)e3`J>FQMnwKp5y zWYt67c(uXk)9V2G$xw$b1uIfNc?CYIbo0nNrpLw5-UrgW8l9Ro1@uE+Ooc=(B5Gtf z5p~);p4c{|!|AshL0{7t95ycu-WW5U^O)Lv|8VH(UhJ8d?4NFZ+Z>LVF|Sds&|~!A zgI!-}$~mujf%5wQpC%gk(edtrqMO|%5sF7pT2SGkFKXZh|IrMvo}D-kk&2%uyr@og z2;8&&ZKkC%YXj&xIvKgp;Y!JU0uBi^jep8p%o_I6uwmpWr{qCRR&)V9B1y(H5t3WadbQ;cS=Bg<6Lt7fqSmdivD^afl#gUEFoJ<=Xzo;LyT{7mJVNtT_fu#DR{CTAlPBd!2Qo-TZURLrz|V;7pTlv@Xq9Qp4pHt?pgp^;DsEWKXwj zD@lJ^J>F}zRH4xSseeO+4t8pp8 zKkA6dw_wc8OaH@~{@a-JZk?O{RFK)Qs0<9~`T!Ny*uJ1n_}&f<_P{JR&1fECD^t2N zKQrNJ4orAzeQ(nV(lXE=r&r_PyVM%c4xztCNS#N|T-TgG|H{5S%Stg-fDwO!a#zy= z+wcjHkSh+M>#qRuC&;OFsmFFi-pETc+sJ#5vXSorCfIAAa&>_+3w9?mcC+1`F(n|{ z2s6(7Di6`9aa{Ec-327VEdr}Uexhftl;I05a~uFy^tlM@0KWPkMAU~TrrPW@0Jo?H zW08IZ2xft*jF zh&E0^=zp4IF9d(J8zak zDXwenu1ZH3O!Q|gQ@G-&e&sqm_`Exld+g?`Ug6hA7ycC}{~Ogs>XvpRuzhHJG)$W_ zOkH)5L3HsXopQYWNKX4k--^6fvAQyawzss0c$l>Qk8K%DL{dE+LEhV|&?3kMC}K{m z=#q#AiRBGby=j7PP|ox`rzt z^up5RE%Ijt2(yvE-#Ke91s#Oo%(RcP7pvck-yFNulsLE8eem~TJF+frJGKMkT%zh8CE>upB#d#b_<46GCpq< zKMVo3ebW=|WzVXUvQ%WCLOu}aBd#!8u!T8yzu$8hjd|0 z$8PvGAGzTJ0lS#NFSM@26{lFigq?H74P~RaGiS9~c4e6jDkse&c+sEL>A+M|PCPWK z^E^PP^ZAWN`t7l23s3++V9wf(kO|Np@O4QmdzO;R>rQtt1t&iVe@yYp@lys_2-~4k zhCDp!V-&coK5JDPR|01lEG5)bVLW2b>@wI?)R%5?l+Gs&w#-7-f=fkBqS+GDHd3M| ze1|`HK4NHh0p!{deMb$eRluKVCko*3Rd7(8f{qEs$0@aTI$O2Z&CAropFBY!KHQ&-y*R@H% z#~gDbmZJkpBd$k>XeWGn!<Avr8H6uxKBW7lNWI?3TS++>t4Bf%sv)M%!hP$R#6<9} zWAQO(6rXSh;L*G2vZv83D(Rq}E9tl{z3w<=qUl&qON>haYH)JPwCxX*BtCRc2W5B{zH5E&s!X=ICTQ0z$2KP4}|uw zXYGdft{t?BCyI!x2ge`Q=nUt(jy!h@oV?RjQRY4-=MVb*&}7drAJ{aov|z>Tu*aSx z#dhJLY*)Y`VB@uiF67s32tU4NG$nf6_>O5e`lPW2L##UtZI-;^Y>uwr%1T5(fAGF{!;ES^tf{j{lBsH3Fuu zz$yOJp5zD2+4jM-oo>a5U?>7LgV^DdlH-|Dk+Ssuw4^IhnRa1wf zhV29AxFVG{&Xzj~8V5=yv=<8LGuTgl{Ktwo{S3?CtN7L_O70ydfR_h4Rng2;sG#LM zWLbC~XcsuFiKZhxMyfdWG62w)nz&!b}pH9f^axhffsS(%O=P*+URwv&*j*1 z;87sWh8}2d&Y@|QR{Lo%t6+Odt9`H50I4A!!*O;Dm_X*_h&p~y^wMreCNl}7j3%$| zw%T^r7HdIR2+jDQ!Jm@nMZTc`)`h@8BmN(n^M z{rxVo973nol|oRmuRTXy5bO2r%9Zv2Ni#M&&|bAUu0CV1~Zv)~&kK(`lx(c!U6EA|}-)|U+ZVK87# zK*?cT3c@$Vwm|W1SHC12HcUedAJ2ez^kyQ4d(vLgeGdfp0E#V+1LPiWx*8@beCN}_ zTi3K_2^*?K=7Sd933qikn4S7yRnGAA%{Hu+73+e2$$cbRPgI3%A zIk=gDGUX&9tF|(RgI@NTjnNcCc=wj9PYkcvd`p9|*gOE3`6T7k$bpoqZJ0~YJ2UBs zAsr;(YN!OszAi)gtk=w3>SO8yMWd8bp7+5d*-bAEB(E-)|Mg)%DKvQ!z|hE%jJR@P zrQ#%iq2b@q?~vc3d9>uie>_ru8ywL-H-h1}CN(L}*_=pJRCk*`8Q?t~4phU~zbmu; zw4h<+ILdOs!c6}6OigvY!Wb`naVwPKa=Z-HxokV=vA;4S#h9DS9{_$O^Nk<3Kay9W z;sFZ=idmiPisjBle>=-dKWhj3v)=%ao-ZD0v%WdJjE-I%XTOggr;_*B7o*+qYgkdF zLBPbC0kgnBrDJq!X_et7Fy(}zQerJ#*s6Kb2JnkPa$-$MsIiggNs+Z0p>^)PZNH=N zc@bvYBg>t9Zh^wFNT;?c&6w@C&lc~TZ`3k*z|B(~pXcg+J7q3AU`TGD-7PX%>ubq^ zyT?q%1JBR<6dxH%hT?(2Lc%5(3ntzl?JBt}qJ(CO{)*+7k0K~U116^!YvY3ER6(WR z|3NTZdCBfFF&8R))`1vwJC?+h879tTxtLqQa=^b%*2dV= zE$j!AI^J6;;J47CF9LYE55n!-3i&>^AHuglQbyPwDi2AAdx$r@@GV#rp7cyz_|6?5i3(0bPr?x)CO*Dqw%cE!A{5^wXJcE;T{cG- zpqtNv5S^KT4@{1iswSW*2Tx+1V`t;RA-so6E*a%Lqi$dSEM9wN;a7O&Utk^W|IAnZ zN3=7Vf9cO9u(zWX%?Y$Z!2nu%-n-Zuy|15yYh`@nWq!;o+Q-zzfGMs;Aoud>D4fbU>npBKdt_jXffbjYO@Ip08hVIdc|IS;ufy29qO7{7I$LIW>Ck_WQ%q z9?Cr=J+8^2kLyG091PS8*~k7p}DrT6qrZi2#C+BCnAO)`aP+jfdqV0-Yes6tK+8Y z*iixlQ*3=Y!;A%R6`$L3qt}*Ik>Bc6!y~olhYd3!TUma!hc2M(B`v@mu#gX=2vTLw zO*a8$_O}mLK9;s~mbec|_<{hO`>_qm5R*qJVl} zmXh~gG~(TCP3SGsN1L_`uyP{zXVRyzTg+wdn6G#H1zUzbeL1@ojtOd&R_ZY7O<_vG zla0S!kODE?1|Yp;xtxwhE^Kd74%z_q$QpzoroxlHZc%TCN9?^!*K8%XTT-1L1acAwgLE1c_=735qlt^6mgp`ylA3)10)fx7veETx{*B`6m%OpRD{yds7eoBo zquneAocbUKoV-$%3o11t7?#sc*y#F1hH6;+byZBWC##}4f>(1Ew*?Q2lz&br6X>rU zL+#?kekQWE+}a}Cd?lo0Po#HTIm-DYJbj2`eK_Y#nMGpSS-=82_0Y6L+Hb{to`n64 zbY~?2(T4N+ef#DsXld3O6Xc^twQez2z+Dg02(gzuyAJ?EW1g>;GLNitLvk~+V0|); zxd@Z}iI47(_66U?e(OETD*|^RrJTW=3k8~6)n7Sglj&Qw?Mglyr_teM8f)63XJ>l) zs0*^`xc7gu4Ytp)81~}k)z_!x!0}OUsJ%mH+_if5=XI|5JbP*Bt?Y~%x|v#U{Pbm2 z{{3>3My(;GD9|Ie@!@TXu@0@wH?Rz5M)M&`#uF9e;0euv@=47=s$nKy*`vKo-ypEl zqzvQr;bAT`$i=BR(p!7kw7pg@BLhhiDA$f#gY}OX(cM9Ra^;`(FxMqf>2b96z6}ar znKAJsJWxDi5ZE0ANaH|FM&%q&U)i&6(6zQOlTZo9*Mmt`x<2(MbYWJeJ$bPXEUPsu z#b{scw>mo$JKldFoR>)n`)ti$D7jmFN^sv-XRU-fexxpcWa#GMX7#j*W|xQ+{O-N$ z(jcEBXOmKE1ctpDPpSuC`L1W(Q73(=G-7GdIm{wCB%9GNR;IdxnvGom>7L_yr6WOcLOI)G@&L$uT^)#5 zc{iS!RBOw&$9&`Kp7WWu6VHpcAcHTsvpB~_e)r82OSC-AW9Gg-V{gE>PbzPTk^gw; zr%2BiA8(BQ2l>4FNcl8z^)xxFkLYsC$Zz}N)y!g+y7SEVK9lF0|8rDfh9)zO`Zq6y z1NX%*y&U?VBjZF5JkWFp?GbU-%ApHOF9BMyKi3>}xjR=Xu^su&%{6IOiiE`h&`8(S z@JP%;L9Uqyji7kBGq-@m0i2Z*59-rr=>7de^~_28ye+MAFs_Jq08QH14TzJ4djreq z8reo(8i06R9Pv#PB78W5=;3Ta);ohB!qg4~R(qk;s2CJjbyZ2Z+E@x7wM zfWASS8{RBl+VxXHnWBak#?g;P>ksRdjT7FKtmC1)uXKP_CqX#3kOe$59R8eHRXZ?iuNw?$c@_KJ;* z{{>F`iNis+4PO&Ja*n@XkNoOi={SjK8GdsPDNaA{``EmC&ELK4-OauL>bbq^q(X3G zX0k*eAwu~hT>--BZ;%l2tK8EwMY-l{+}x8vmz|1QP0D*c530agPL_WG_-oBbuUMh6 zeLFDKUt^0L*NuS&W6pg5)OVr>_aj^PzG%WD`2GGDBV~0mEJJK zr_5l1YZcH95Y4cYHWXO%i5v9R-)_VPBeeO$lxG#0DWuNB`Ygee zvsWTgG`xejom44UkqsKOWxRm9#uF`QS4(IKTodKUu|n|s;ohC**NL*bswUU2xms_z zL_RzqEf!|o&*5o;EtVbm>1IF{Rh-nUu!?al@JtrEDOP8cf|>y@vIPTgJJ!2g+YADy z^$Fgj&I=~@XLE~G^n6brrL8|v?D&dVz2)#4pa1iW`A=t&ztZ@^$!*jcFwVppH-E7g z{mU)h3w3@%{O-4QaO`}sVEmWK&C}2H{r*jFBQep>K96xD%S}x>ZvEhbS=;c5Q5{J? z2p_8qYcU|d^D}Z!Vk4cNc%BG3-;3rbvnZ=i68>~;)mD%Ujq(#svBwgpW$xhiR=JDj zoV|95L@1VrDNZYO)dI2b1ar!l0_7h{=1TDTweCuwt?hrv4FU`+=j8(@j>-oJ-#Ri% z@*lVJS+41EikmlgSxoIuxIOlEk-jn`AXy(XJtmC8@51ly1nlO`0-Hf}#(kBOUUzHs zLR<&Q*=O`pdPKA(PL#LGj)5qX;@Y$fy8((d`hQh;OaA zG5i8mn^2bmi?tzDD>PP{D-S3lX#rzFfXcRYC7{3KXr{q|4t*AEUsCLXdM(@0?{dG_ z$KAqzOrBNK6Pi?3!>5UG*+|gWtUIvjzjh3heTHM7x5o9@5AEED3|u5k_9}l8J^KC@ z2EVA4Yfnc?S%#W$%sT$ol8 zz43TP%pjhoCC8OG_+KB}|uZOsG-Y(-g2MMF==#y-Ra#Om!*4 zgA(d0iqfd%ltX78*x)~cNQCKn&yAvs;}&@bW85ET4y6BkIL|c)6Q16NCF#W`?o#VV zRS-?1i&QP6ZwJ>9PHXGg4b&f0`wtra+dHdy8rO+s|2$euM(RdXqfy1F??k)uv1Z&(NRg3uH}m4u!}s@y9*c;9BgvS6 zuUV6&GQcRn@&l*$V!8Q{n2E~V$i6ki_aJ)ulbnft14bmX8-5V*4d1uNZum7HyRpBU z6yOi2{z6Wd3Z+3Fo(2b5==vd+v?138Ih4DAbL%_0YQUf4DM zHqJLGJ$W*5cjjbLX0mURLfRg?sQGIfQ`(xl)J>unG0KViIkrYopFgkpYtm}AZAr2W zr|+w{K#6qKBDijnwKG_#;5Wxr2ODv_7Tl}0eu`n?S}gJuqKNc}Ypp!7G9&l=FvZKp zPfO3L>2L<>Hi|`W&|LeP+T%fYJ$6nqn8ZtU=lW`~=s0`UhUgJtma=D{H**V)mkzn@ zp=Z?D@@K!s-3E)R#e{|G5gk7iV1Ht-7^P)PBSJI;KqZ@tLsEx9Eyga`+s~qpepR?z zo=E%iIyC}V%dvhRNP@y|mv)|A>C42L^jPp(o5agq%tZCDWV7(u_KD{3Kw2xRkp1|~ z!4tqRTF;tmAIv>IvHs%%8OS<$Iw(YNs+a(bSDkt6&BzO_N*S#WrUJ7A)a5iIzM8Ws z85uEOLVx+A=jMa&zye&s#%hI)rhB9r6(md|hbo_v!&(&!mBi9t?AlKOItBRalNkZ^2nUp{q8Jz#M6t_pwW}VLXa9<89 zv|-#UmR+{o2*v?J6Ds8tErmJ!sZyD-R`gso!xbln1q7wm1DCVV}{T)u71&aOSshH)I*^_J>0iJ5ph+E}r!- z!zfowb@aCf#c9IxPyEqxpr-nI)X~>eAvHd1eGw=g3*#6j1c4i&QJ5pG%-K z_x>p?6oTml6Vk?%CjYHHH#HlS3&!qf-Ev4h+VS`7)q~FdjVh}{3X{?r0S!(4G5;mw z7l8mL6YvRiiBVBBBV_{^7&6|Rgxk!)h5Rd_jzd0i+Tqxhe1*TryH=`Y!fL)uN|&)G zY}N+dHF3i>vK5<5ulgJR?h<>;PIy~|{xag0#rA`Lx(!Bsc4UMXi+qk>Z8#?7@}yyf zQ7<1XHbLex2rq#LjyL(XziciMfWIkyJ0&|H_7=~EY~c8V*)nhu_9SCoPilo-IU=TS zd~uik@%yxBfSiKcbWZWET3U5?r2=cj_q$sAR&90tCpgoFpTF^q>DmFtZJraafQyo0 zmeTx<3Ye9qY-3}fkZq%~B-`%T;6y!` z`UhpMd{MmK@Yc}`W*HtQ=2Mx#BH&1t&_+s?%_&KG_q#Lz-n|R5Qx_)lR;=uX7GI%* ziIqB^O!it;RvO50wM5t%JN{<_3S}N=0(Oz~F`zTgYczDG+}{|b4TOO?bxOJ z4-jLZFPY&xM{OhzEX-xq5^sP4v;!Gkm(=p89$KE76%(et2R#li{`5kH_2s;Cf2c{q zq~BL>i$St4eXR-^nJlDB7wJ*kYVN_$sJFfcKfrqbqI{VD8YcOxkna57rkh4Kk&tb%Qs8P|~i4>)=!wSRT99wJaV! zItjD6oNV@J)q?Cu+Dh&`xi3I=PMc3gl_;&kWT%UMyDs|-vvT`=J6u1Xtvn-8CNJpQ z_pb37^tg)OEZvqTl-l>mhL|5NS#f^}srN&5@>x5%2>ee3g*g>FwV%rt zbBVEjqLS#Y7lDj93wVY*2zZ4ek%T=Qs_&_wVvd6aBc<$Zhoos)NmQgs{*-i#xKMgogF0oX}4ed%?vp zY1;}+nWXiw}kE!O#I~VTh?S$A56-(FPjL+2Ur-G-f#uHZO2p3HKamy;teL|C) zmj#Jb|HAbV6O%pbz`>7I@2t0u*tNCuk4}z|(3aBlPnSn~C~PH9)n^E=y(M;&c>9LQBKz)+El)?%7_vhWpDde63>=c`Jc@VF?%r0u-tcaW;WzykUn zqZQyWH;OBhYX#jYKxeEWhY@4?aDt-JjV+hP;)H7{&j^oBA#Z*UT}F*^5mWxf8NI%y zZ&$R$PckPpsg_NL6-4|JTWJmzag7mUyWFj! zg!|?1e(F7Dn51#%qIfCEhLaW9dG5ErQ2(|<2P_W!bthNw7L>(z{shH0e3HsGZDbX8 ze#CLs&8KTRWK>;g+Zbz$-NG_364mP8sYIcf4$=Xe{pXt+?hR~<%>S3hD}VaE-G)e4?IXL0BFBZ|hoJA~m$qP~DfTSFAVmw7%Xid67xJ!Y$fx zwJvHGzl>fOc2%+bX4fx|^%bLkU-rrlPCv3Z=~;o_Ju*?a_G>^9Vd2yUGJ8F^5fjpN zUk}9ldCUEi&9Y&-)Und@-En{hbhKb!key99GfW+TnIB#kcFHkVOndR!Twjd}F((7c zt$5AIwU6}-3(@enEtWVkM$jr9Aqm%tMekE{m!}qT`BdO#XJ~NyIBBC#wV^k8&ZY^+ z!o4)!UTNjmtGdMShj5H}sjj}KAU)E)e$~x6Uj(qLk(rz1L4jnx$k;hmiLsGm#>$H2duytShNudAywZG@#ZR|(l! z77(*6_B~1LQ}kir(G@@q42Wr-CeJ-e^?QU%u~ZnSWq^CZyd#DlR^mX z)VR|sH}9eBbg_hXPK=J?q5J2t(&!J>sUU_Cwmejhm{e)wi>=mX7d9;a`o3jZ={r~V zd;a}fXGbh7IM1y^n&v`^Z$vce9pbuZh}k@>iU%w_a@RpxAZO9I1h+a$%wpRqq-Oe< zoXluqmjn98euTL9O2;!rBYY{2^jk)Bw1bA^Jp;;>b|4nEI5I0mIUOT%SE!XDZgTjn z&mNHZLRT&CHB3XY6Cya<{A`ObN?H*U+^7`->eGwb92sX|272+OJd&9U0gfu@-^>%I z0wnZphs~-`ltlkTuEMub7YPy9#8#;46jY{11T z-H-Ub1yI>$OgmV;R@Un{_G@%hUsp_naK>;E_GWD8=v&BgEt=6# zOw**Q+`+cTIWj&aJ*L!#MlE4#xZSoxdX^DLCjv6ki4=_0K=8|;Q+B)zYr-PUAv`zx zr^0q5a1&_YUn3grr0hIJ+{Q=#FfatI(D&C%m*C2b&2t0S{{k3-jHL2SB0y*wKjkqssWiN5aciMvvf z1b{N+gyvm?$vImw*b$j=mF0L_w;P3mbH>WIT^kN_3eA0|rQ3EZG3=$W=<3-dfY=tY zzhXGwBw}{3^;Vs1_~ChPl2x}`M4PNxd$@nwekyrg6z?YZwg5UoFIV+#JHDs?oGcF& z>F{4a>*oz@4FFWxaeO@5dcQ>mhc>k8{(Tme0dYO5e-z`91k;r}0Hi6n>8Y(=^A6nu zvdS0t*bb`h!eMVs9$hn{5%T~4z&D;18YHZRJ0(8)*m_|u6}NBj|NFQf!eFVOD?idS z&U8X0oG}K7#PAU21)CNNs zVEaw`T*0B^Kqw5xU(k6yVV`amdf~sfd9J}XPkMhfS*Cud?Y%uZeFnb09%72Qpn7*Q zwccH0{0_5Y280NG6Mnqw0~1Wr+K*UlLGD?^AM9(x83nM(LKBeq(Td)9l63dcp59;c zP$!Sv0PIL6Jr(aamGrcGU(JD=%ostrOhAK)`(N5<0Z#cVgq7CdKOAfJhdp`uVOHS> zlr6u-h)gcYxhAc|h>#}&N9b?ri@%@vEdk|dA&I-z!MF4Dz#>rPP?_hR_7p*^x|;zO z&)sYf+_rLAUT%FxdA6GT`e$@=29^k1&L)*KPC|F*aLYM9BzExWTjvgUV%$xOfNgr? zmx!m-{SZdwdVi3 z6z=I2!5hAILC0Lj_!H^2>W;aNk=bqTlfWQX8jgU` zl2?TM9SC?Gw<7m`v3dFXxu^(>cVNWvH#?|@;1={-6!G&7_h8NUnh{@TWT^dgrHbnL zRz|5KJ%HY&%dn)o1Ra$w=1;US2>wHX>m6(J0L!d5sm)k-JaW<6EJuE@Jtlou$NvT` z+9t^+_y8D+DA8oGS#Q!j)Yl{TpzEySamQuFV>UuJB0%ngz9F1lKI{o5!sgAJ{W}`1 ztLG2B-Mqp^O>loxkwT@%$3(>iB;7)ENoMigVS2Yk(i}&uZsU!Pqse~tMfY;tyV&#~ z^uZpmaeokQ1NBySs9b`FI(K8tqKDJrgZe?M&lUss0PsbSt zjttMz4I9Z-kpbfk{6e-k7J1LI12COLuBJ(Lu@LeY`HcPi;5AajYxk`GitlQFI=PUM zrzBk+p)b*f?L?!Xb9w9fF@6mJiQWB{_O;$N=KE4CNE7+?Kq3z=AOi}~^KqSPUoX|6 zm6<_#1O7-KB8ASWwB-N0jOmH_b<+?nKvAA}KNP*^@pq_!Ftz6uIv$DDBHjqckR|iO z!-mM5q}5u9EA*WvR_tc{n>K0lT^hAT4j!H_xBmAk{ogjO@xyA|o>3gd1E6x`jtqyG zzOb?5ph@~sjz9F9mbn(uW~;hd#IB_9BF(fk;xqg9Ed_Dsh;KirNa&i?RzBnCeI{do zAyldZwi4YKSjB+j@^J3_{47^$qKd6y4+phV*Vb=rt`Tjk`(DL#zadH1&^d` z+h5obVuxOumvn%3W-NZaKGFU4968=n*luiJcg?VJ|KLzDx<;6~vZhV~D!_=C3~o?w z$z5ybsCmN>zk3$9_<0wpa==QqnK-z@dSmm`e&U_mI)_9KNH}iPZzTv9B`pLx(ZaLM zbD?GC2koD0tKn+B?Olx-hsz(mohJYSLy&(d;U4Bx=z3*SFaP{zOybF{Pu1aW=c>cT z%Ja^)7N|s>!_Z4VWV9_^-)g9aR_e!Sd}xGQQoI1)wk=@QR@XV+NCT3FzcGt0 zH0agOLv9z#2lZ3ao7%3=@AayItH`k3R@~>vOK}tJq@eGCt0N5_qC<-VnD$$*QY^ zvF#I%4vs-?`Kzu9gnDMM`*Ie)0 z*%N}8j;9vs&kMu$7T^=-{|u1^W5)AH{{vcn|1P3m5$Y;A)zXlwB0f=dm3~ZS5N#Ys zjt-Fwr34Xp~{~*M{K%ZsAL4c@#;0-i0_Vk<_7$nupBW3qPgM zf7mXz*P(hXiXQ+1n7>omCuTIVuh4i-t*IKh;oHejA0Qs+#PtVKH!Gk2yY2CcPLn?m z8aY?zj&U&5@nU8-YaN}tl(I{|2bkBf z7&V^5jXLw$Py!qtSeIYH#tb!{dIkW@&pPw4_f`!?!?x`T%j|sD%1W!QubF~2{t*Z| z$)1K^qz8B6?5OBW@hIJc-${%Aa7Y{*fCb^$j{mZ0GM?@0(qkFFWTH&Y=OiauI;y;H zoL{1qmF6WPkMkMjb+C$O4d0@U4r}G^rQpm_N4vV*!E-^usFfKF3);31i{##6?J?a0 zSbrm?ZDMmfqjisQx}>x&O_z(wVA6{L5Op^!Jp{`*-o@#rQ;wY92Vm@jdmQQe03vbN z#90y*Y?|R&v^O%MGxEJdic3GJ(Xz1%04@G_SS@&I{&#Nuvr@lp&X4?b|3l*RnU>dt zX@k(>oK|AuCgE`Xy*TOoA(p%In84V1JWK1w_QW&jl8Tf!tw| z9Ma`!c*V7jCyLackj1z|Qljw(B$uPs(N}=iI}9GV`Ujx(_^m0%`me=y#4gO&IJQx` zB7QuAtZ#N{}TEVkU*8&u=_q-=fz;A`f5?CX6wdCjdJ{JzbhLu z)on_0Eh}G!J+5IKcWN_w%wd$&;Q*E+w+#C$JPskwgs`5=R)K#r)pDUbQ+|KnoC7cJ zS6^jrou*{$R@l@%pRV`t8#Ps{oSSO;YpMBY>vQtgQgXBafUEk=hZ1@Qvy?%hXsi9d z>yC^tNoWzho6CG7?fO8(&$B$;K>EPcKI#1P_^&b<*tNbg^KojiJk%rCx04y36!+o~ z0Tw%ow#;XPZAQZPSowEn1I#7o;8`{6hDXI_|2Bl@hxe)k-03 z-JZO*vS?0pTm)(4UUhFgn-vss{MKv|c~H)H zGG4pe`{PZdrep$I*X-FLu_;1Qw^mSz%pnE@bH1#0^`m@C7@5VucIt*vMu!=++X7pa zo-U|FVq4R&bbd%x19hh5RGz1r=fbZ7eV3*=@Oph}y$;|Q9IVQ(fyd_)Gk0x{wN2Vv zQ)s6aK3NKl&|3Hjg7l!o&Wyt-+9E^i=q|ly6d5P1b-e3G#6)n9MgTT|cJ#TuAAZ=l zcD68^ZNW)3rZklo)(n55Z3qwN#V(e4KPUQpv&-Vwv56iJHdz_KhAVfE+Pc`BKR2au zoLC`z_&08tVX2Fc{ou01b_I1-7a=3p?Cu;68C>1^*3_=b#I}yL>1sQt`y5bXWC< zKxuh?n`Mq4#M12mT=JzyZXg-ZUH@!d+>9l~D+}exJ22P(pGPRZ@ei>w?7Q{*JXfTz zA$QEbVxgE!xy}qKKIJTtl>BSPyS1{;x`eYvak!Ic@9HtxQ^M`X9hDYpEWvSjNJb*5 zxeYq+k7JS>Qq5JY^hJ$XiA zBJr+eCh>BAIoaQfQpsLzm2&6&SI_p|SLa=CzV#4%&F|gpOw>#@c<#wI6;nfyRyEkN z^*M5ooOL|LJE8y&@U=unL9Ebp3UM}Q!oJDO;eA?RK4A6@;)mwerv@rt@=emBpr?Zx z(vfD$+O$9n%F*uP6HbN47SHY4cx-V8@l#YVU_X%f)Ol@$S2SzxApYrm{gpD!~ZOaJb2bQ|F5ULPm^CV z<-F61TH*l6`^?gjwP!Mw^J-=hej1~dO^ZGZ`R_!aPFH)A7`9{UnM^G zMF|E6(qBIi&Htz>LoX-G0IlAwaFX%>CpJvP6fP~PB2DBF&0J=CGd}sF z%!fc*HCO!aPv*z*-+)=rAG51waW-m6>WlC8mer7a=LCFjOA1YSL$6>x-OXb9x};@S z%Y{$xC8*6$P7PW07AEPmSb7H?j5i?#xg;K}EcpS|EI57uaagTw2b~=Y>va0-nzdb= zL3~^-PxqJ9Vs2GQ`RR~HnRrSW_cmqk4R-6;>yht;9n!)P_TkVj{3QtIj`+Xld6ZMS zIKkLy-79xIb`c2OS^XWnCkMa9zFGi1Fwi9@>F>|EGwh#NSHl!^!dJhl6JuExXLVc4 zsq)xlS|N+YE_pgJtUh@cSmhy3_i|y3{>;t4_KNtVjNVMdL@&WyT^1bro22al2<6+6 zbAiAu)Z4iW^ZS$#a+>Ah5+ekXO9d3hk3qCu02ULn7BU2Sb|pa7>krWC4}DS%%=G|; z#T6IitL}V&36$4H=DY1H3BqSN9h6@ z+ut?spFaDx)^A8^!6+HL1q3WUo9)elnoks+v@v1AcJN zz_#TPUCR0(s<7g{`I|Ckh+62c3=oTRsrol_H4e}3p4_9e#+ z@8`=WKiL*XZy}4(2jeinyP8!wf{IjEXYRvV5AA%z5m(R$&EN9DjV$GJ0jy&#-$2KS zuQ#kNP1Tz`V49kfFi{&TF5s`Q$Ym>D_OvsLn2K%;fJ>JC1<;Pws7gT_jKT(@K@@%% zKXJy5u$>Bljo)1lZqlbH=do3kSn3S7uCn&ejM@8QDXI_(YiRJ%T;WlBGarcmA>>9##H)Q4lz89@f2n zDP-UdrtR?3yv6R!Wt)hjj;(KXYzaPc?KjN!e}4fm?~mhM111Lpiz@(Xx4DSl4vT%G`jQy}V8D57Md2RTHe{DNG>5jIoovLJDH61aIv*B%HRqJ0Gp4#53$zW-Pu%-p5M_#IRXdnl zaa~+UH)`(Oo967A(pQA!N3ixgId0Ziw!ag=RG0HXK&&@j@of>W6`05imDpO9jcTB)oW6)eKr;^#;gwj_~g5b%u z)F{D%Ar%wd9LdDjsrS}$y*>u1sA|E=aHOn*d(1U6lPXUXFq2sNOsT~~H4(c*M%Rlj36f0kBHhkLc ztZVb@XH!J2!lhltn6R}FYQE{#FugA>bLW?~BA)F^?^ecxxQU3h)4$!D@_fdCJ@Ke` zJ)~Wx7WTJhlIBTHGM~MrVn*LuR3|Px@zX;HF@Drua%PM}S#ZDJ`e2dbi7=p~?f)gC z_8{cVif4DdL#fl4jS3CGvTnC&}_ z8G1v1Nn-B>eri+*4Qx+Gc4(20d<{=#$l8qT-_&(Aa4thTMve?vEYw2phh$~+GvvIqy5zOl}0Z{HV@+W z&mBYFQrw<(8dpmT{>^t*!rndJnH31^>2cf^au}y%s`08oo8_&WnsT(Pi%#K8J2}8%Pf_lYuUtywp zn+iqOhsUDe1vlVu_u50{{QU>0hx)2r{{%C`o@R*=Vug0Fipvxuty=PBX~!qKa)?Hy ziHd*QL@;5pMZPy6f~po_h2HnudT$ z9>=Q%_PN=q-Dgj)w%`fe8G^5{X(@?fhh|^*d5Yuz7$@@;P+ztjs}U;hELzL+tx)T; zR1}k{CO{iyT-JvbtKdbrou!VIfzR%;AMEjEXY-?AqhAvDD@X3VAKTHL2M6-H?^2TH1>JS(H4Sw94dZ1er~~BR zN%NMQN4(TcZN)OD12(H(4>bn?@b5c3tHkAg<#Tb;@bz`YbI^!(w5vO@8+D+&-yIVH!y+2JOAFD zO$m=P7E&3+?ZQ}O>_Kszk6x|Kh4&|D5i2y73GI;ncM;}>ekRYoc{-I7AElRahTqg> zZr$Ztr*i>bD$k|<&a!-}DYxYOaK-i#(F5jbduFtZljzkZ=h)FkWkuZ9*%*e>${mSX zYBBLvTWiPcIw>x{G9Qdgz?;hV@(|#|F~2?h-mq5d(cr#-@_7~){&wlEZy##<;ah2A z0Z8Rs&ugQ@1azi-=O9W(gOx8nLE2V8O+pea9d+;OQhHm_Y1ePtYw+o-S?;|Y^X;mA zw~BHm?)FyZBPd)}a{+Q&!TD|+C2}7W8uIDU_}&EmrpAlB<_OuskmS`3BQ$t%&EDSg zfoAXD3J8}#zy#KAHzmZSb_P!hN*F%GLP=o`eMbaC6aQe_Hb5NqH&1M_ zGXKz5K+i_xEFo|=+MJpFqyqX{9OyoU9DPLES+L#-bdoFaZC_cG3*X8xxT|~@W zD*c9+=Kb_JC`Nj-?3V})Oj+M!y`jr#$d^~Se2F$&TJI5+!T>GEi@8<(fZQ>rvN&dS-@3{=*sPmZtDrTuQr>}Ji z7l~?92Zncmtx40@$01n^Ve4b~J2PnUjOa){#fg1$&;k;mYhL}%1e|FV@7s&cA+SdpXe9fU(4qnp8*;B4POr$polFr9O zN*me|DFC9#2ninNcv)VlsJxA0Q; z?m@a>u*-eN!{vd08C_=9E><>0xUbr^!yH)-LgF21+1RfKa7< zul!hdgB(r(fsVPeWh2>Ux7Jk!iYR)m5xh&Jd9eQ%^gt>_t73$)=0(JX8=f zB+pmTfhzT%5b-jTN1u);+sEW<>8@`>*Z;&SE<>Nj$Qt@^oLX4wS&@iuV;6@07k@2p z`Lv55j(bD|>gFoIKHt_3h@&SCMnBkb+*gV&kVU0HKyR|reT@(ZT+|24Go4{<6Xhq& zdPGPzrLTmII)~4vNIn_#>NQwEpYr>oAF_1n`x8h1pD1PcRZgqc=4=v%aQD{zb@~uH@%pwvesjqubcJB|K7A2Q;6;} z4Rs}}{dxpj?6u0Z4=bU9KUT$Q=_**9O^k#GmVTHZwMv z!B3koZLUjfc-;0m)#co=>&OKT&>TG?D2-1TP71zfH0ez;O4xU1oyqL$KZ`rS-CRb& zT;C{KEJw-PDn^vN?zw<1Whm?y9ER8Z9L4`9!fAV5S%hhK>Gyu!T%kj^E+JfMCUN0Wr}Q}^sn{q< z=juK=x3NS~w{EjjhB5sgPJJXV0spJLE& zyY2v5`aL!6Ti{J^Nm)O6{rZe7lJg(-ZF>q=EU|&;N*q)anObcqTwZIR%w6woAS}NP zlp3RL&5#shKFM)VWKQ}SFp^a5DW(Q*$XsS+p7J)Oyo8{7*}>JGZg9_~X0E6FzO=1<@oyuQy2Im9<<%`ET;rDx z0X$~#W_3dG$x5(UP#~)MC+`@sqQT>{=c&+*TSiR**9uj~E7+#xP+%_SChw}GngPkK z@u)e^mszBP<4XR{%(pShYCgAe9sgrx&g1fu>4aDa8;CzS_<*<9Lqk-#c7eVjd!O1x93_J?T_4@ zTo(kRmi#%07xz0rddU}%F1{n=q3?Nhvn&HnSyjhPYG8u;Qmf9>42Kgy z4nUWq)pf5#NAY!mTRA{cGr&clvdejnJC~@DALv6oIPN!Ux`IC|&EF+UvYucyrEkbB z_i?rM(J7~sS*>#gJ94q{KAkev^*=%$g7GQ&zRNiX6@VofM%v#B9@3@u1O$B_eTABB z8xU(-BSM&`uQRu5tvD=uK$r6Dp381mRX5&J{bVAru%Bu*-9KrN`w4}b-gx&yn@T(F zpfBb^I=hAVAf8DAbn$+eu0KP}Zti zo*?v2oli=a_^xK0EVM4ncs5FcnEuIG`n_`rZ?4f=2HKFeg@^&KeE9E|n|#iEIUJTQ zn7-Jg2)YYAxE{$k!jhwCzY$#>(7SDibL7={Nw-l{=*U2`NvHSlGuw@?Ik7*~H|n!e z`FeQ@*-s}HE_?hbJJck9Vs^%mZ=+M|Ofau|_efE%H+!Ercxsr*6*&W#pr0MT+(-y~ z>%Y^oR-1p-Gwr%zu@mUqWD&`=< z@I$(j6VLd;C*KY^@(r>Mw4U*InEG*8nb{kIPK+qf<6l}*GrD<2*7JWGS))OJ#{7B6 z(I+c%$-cklBRpDjk0@0P=|Ab7k^3=6DM0`y#gTaNQLZ#0x27yyY#-t2wmXf?qx@DW z8asT$-4o6j{{0jwC`q#rcca)^KlHwE2H*AlHp&z4HDQ#?Dqj`slxjT*{aC8%tHtxP z^xgaQCdhr+;;4}v5c`^wAqR5}wRo%?zj67YuVt*nQ^Xbo-|8LN`~)y9>p>l7&X z_q?s~W`9LQ5r;XU5BvLs&!`bZzZhF!hFF5w4Z>mjiUtbUp#c15#r)-g}H*ht#^wZIX2i3+ov5)$qe`EHC=8 zft6RqETYSnaL2)aWb~4X2ln@8B(3y#lHHkX3e(`k-ks#rQ+FBJ8#S9(!HkM7wS|5a z&JU-oj$~*He?0^;=}RA_BS}$Ka&pY|TgxouMNlH?R(xB66|}+(jYyU7cscBXD1>2C3I)6?3eONyLO>^$cn$l^vNFi6EIA(*RYmatt=gpfy~yl zXmEf(%GJe=lQphUkgKXmrQrI;&9KO==P*fUj378*cFWeF+$Bl7v5v*FV`^b%Hdp^O zB8)H|V!&M_?N#ddaZ}`cuc%F@!Y9AMh`X1kHbKE3`3g6d$D>fqYD_D7HZa^9*wTNw zuGYJqn}!O!$USBCu=Inp{h!w90kfCfV6zv|x1Ebq9EX@%GtQq=O7?#QYWZl14(DVI zy*AOAO{NSA8s2W!q6)RRtkKiD4vDO%Mruv6J%GrB(B(JVqF6Mv-^OFgIHQz88ErC? zXYM-jMxmH4PG*gj z(zSdEQgv3~QLe#FVkQRAq$Jp~ZAWX|=&VFG}#1 z@Ci3dbb43(o00ru;bzH_+p5tugnz(Ps_#S*=%@VqTjz6D5^AawY%2lmUe>?X%;ol$ z$sdo(oIERVOwi&d>?#7P-{biI`BSSnS2MItN!$-#X-Q^u2VBN{dQPypF$uH5x{XW{_-KW#c=}0&WM!Iaj0UR|~6&cO} z4ze(({SBY2lGxyW9{1@m?KcIyeM{Hr`c2T;zw=euwKvRfXmhXKGEl#NpJVF*GD8^tOM%8lS_wS}cA|+`%0+xw; z*D{tr!t_G+LJcf}U9;<4a>S#Ze{2PN@M0(ms1s9t!9RM`f`?A}OB3FECNh^=M4gR! zovWJ$F*tV|4_ue@=P9AfUeDUx4dcD)ufr^6zWR?s_e7PuEG80C4bN99F|&-QR#U|j zv{F2Xxl0+pN0ohNXWI-WsrBqOtK96RM9)s|So^Efh~uwW)nLeF(ZjBZ8GPBXnXGHL zSNQ{UYuI|QS5mGRKA!o&I;7XkO_j+GUsfS)DH=>~xU*yW1v2PGehBmAiG8)a&g)ng%p)hxP^!@MyaQww0FaigO%l==H+b9 z0ou1=Z~B;(<5*UJ17tI2K{nyS_$LA7c;n@)tJ89*c{t|WqmG9TQvao~#htvK*hw%dyZ7molV9Vn zXHK3Wy~0u7^;236-v!(lKxYaHUC`qM_j`8`ye8y=$EC+d>Iz>gOrG>jn{N#avTo%* zE>81*C?d5opHtYjXKMWA^Ftje-FPN2*QY5^Sngc;<0yfJpw1s#_O!p}xG%&tDJ}Uu zcS|W5J;iL~pH25j8B^4QxpwTv>9CH*YI}~# zTTYWV#~XX!ws-`%RqFmgy+-Su=?Xk?g0XFU_s-+77U{j4!#i;;6~I9SSX_}Aa^y4T zuRYD1Avr3-3{rx7KE1h{JRXSfoaZVbV0-c8`Ol01xZi)`Zm@KF?28uG#ZbF;uyF+q z`TeKjsikTAF5a)^EVmHh4#=)N;V*OCrRHmb;Fvp?oPDmkYaIU&xU6HIr&WG)f}Pgu ze)+)buJE6i7vzs%=rABm>Z`8)G~|5i=F-3~@y&IE`KJ%RUddten*2Wi*2gHpHF)mL zXt-x>eEDUh9hzf@zLIPI?QwE;T7H-$FcIAkfpRO37E4{%*fevG`SdiV0n#Z-Lhw8D zC=A#N%MS*Vq`MJ0%b8_6iyQYh#RjM{Ti{9R%dPacUxpQa(i$aR0VaRYuM_POfY1EQ zj}U-Wyb7n6D=?IEbtz?Y(t|V}3}4}(u(+0SgUbf1%SZ#rB$D0WStNmP@^Lsx!_>s=8K-pOd$k}^S)UJ=cd>TJrid@9-ZTizgn*=Nd zR5=a8@e*9gUmnDUE1>#BsM%VZFm)x8T5XQ=9K=jjWI42-=_H2Bq~mu=6#OTrgEvX!$#Pvj;r*lY~r7p zz1$ofj@L~-x)6^#7}-PbEx*UIx=!i-IDb0crT0sRUSrBDzubC-rxljHE$SiBKKEBl z@PiG>9Q^?S@=4t}V#brjr4=4~imJ7P2s7oCBGNB-l^~cX;)ual5^DuLH`VguE!>Pw zhqZT^=`c&euLdE}CVS#^jR`8?ukI5d0;d(bq1t0>c9X%#!|hea*EMV&K<4NfugnkD z|H~fPUBk14y8Ns=0qQhVEd4Wcve0_jX9f16DRE>aFMrMnwJE!uChQ+yC~(2^-0-;) z^*GMObr(zSp}>mmvL8z%j^Zl?2CN(6QZgAATIf=gw9_DC4WE#KxYew?y?LkjmzDut!*)WblE*Uzg#o-&*VpCyc!Xk94ZrPq z7uW;N4S{R(omWd&$+zNk{zRO==M`=;Idz$-V0$JLy%qN-duEE!pVpFGD=JQdEG%>O z%rjrUTMWxjOXuprlspt%J3uaN(J&$p*H#nBOQFDIJL$g1v&ha*XL;9D7=RZ&|DZ zct_*KU}oX!pL%}D3QG2DN8@On_a{7fVaul?IhA}Q=VZ9YuZ7y2Lx_*}-Y3<7ywaHh z4Ne+pxXy=II1p19XYb8uoLm%ppGSE1vckr3NRir|ePK~1M>ViA1vWhz=5_HgljJG| zTjBk#CUVQ?QWkZQ@NBexxxzy<26A+U9bD`75C}{Wf7!)d^}pL#!24$%OK;tS zrxk7Ds^R>GQb%Tca6=ht7IuDMo8-}Z=S;Xp9tEBr5ce%mS_xfxZ<4`zMKbAYfKgbL zVp@QiwR4dt&&=YG#e=9qaS@Iy;c>>hec$suI+ zO4QoR3cHOZu^iQv3?nApGba~H&pfIj-ujQHMn8|3?X@0!KtSUBFqwbqyxV3&JhZ!w>pl*Dd^LY0in=(H=wnpl zqa%aGHnUXJubkTw$tPV)=7#1eu7h`T+X~G{S%>>QVB%{1n~v|)gm-RQy=*k&_T<5S zpHt$=xBTxQitjW|H&k{7Vi4g^KC9je2Y7ggnT{$EZHhGW<5S-QYi}RA>AD?yC84)0 z^$`s=oIA9Hb<=xCCmUv+CnQ(fYCkzsX&kKwthedOPF@3{y6}8Nwj`Q2iy9R(s<15Y z9==)UlJ5vpQ>D5x>3>x6V|_}m@bkFszS%DbcGVbo$nVwGgyD1daPp1o4qsH=e$!(5 zhBltNust@cKM5=-)m<0+`sfB$1L zT7^txnznb%9SHlDD)_*4!1#F~#1Hi|#^zuCqf4`mz_Bnzn7<*m?PJxZaRpx@p9^nl zd_j1U1H%>!Mq|6Yq|qrsumY$qK*9__dWk*@|5$V!fI!mt3BtPc%93a@`gP zjf8lW&Q8ZWHoEDY_ClM36ah1k_p*0pca(0*u8t!pZ`95yf#;q zR-A~NQY<)O@kja>Y297}g(EIYr`o@wHFSB$I)wDATGOQ!Riu7WE}PF$yB?iy9X+}B z?G<>6!^2BVI1Y3+X8>GVN6IWv5IB3!=1)qkxK8m%%WdaX2ew+a=(?^EU=o^B(5d&B z7vo&k_;Q=DDa+id*o&DQAkekXMX_`EhL5*?Ulw;0<{O%uO1nPNzX?8IH@lk@Q9@9= zJ`)fif(l6Iqcr)8J7>g%WtuO4hd38W=I+03&UWcH>Okw;IS-n8u;gsYqTu0LR`LOl zAjXF6#WM0PDg&toY{ccu0{a)rN&Sl&?<9SWr>$(3a9M$So{|`PdIvbA5!?So=v{tI zR>`e;!u*@rvZt%GOhKZmRQxp_oO z2;&!Cs!}QIQ2jZbx}Nmi{i3fMoigjK+!fonjJds7#H-KR^&akVcd80uCC2V-*jt%3 za%+6#c4cK{%J_L`51%8Vi*%XQ<5~F#<{*|%p_2blPRBVk_#edUKDWyksasZ*oAjE% z#Gj52b0~0HvS11t5To_J|NeSOCQ0%611X=z6xE7}(9M~Hm2YM?O%H_YC0DpspUu>g zvk#s|yzO|!d>EF@;`?DJQuWpUkv$dE6n!ML_l74;Cq_c7{$?NrZx+ioKhmUhsif&K zYK_&OQcBcGf;V?({A`fYvSmdg8`3;Px-Yjm$NNni74~DCMfe$J@@Kb)^tt>tu0Hv~? z6SQJBs*H9YK?fGumZth?@u=d$Bp#%fUrP0RT|w0vSidlm>^LXUFLMzSMU$P;?%Kix z+Vy>{00A1Pk+*}A)cX;B_^efClq0WOy21F*I>4oAMyytjIcS;u%^p4MMa*5fzc_oQ zy3CquMA42AHc8|?Q=x6T`ii;2%^01sCR^m+^HgaRR} z0k`={=qM;e4)BG{iSyFPYeFG%-M%DSbW6GA90J!e^YL|~Pspx5vTe$Fah6AyUkZQ1 zZ9IPUs-<(Pcm<5s4b7=-;(r%F3iLrsC&A-O0$+1WOAo)^rk zZg`~i)d9->;=sog2305j>BmbbpDzHhddQ6psN1iW=IvE3wc;FdkS zFz z?LXINt`8Jd?GKW*&N@{-ciojg8y)}BHBM8`@P*2c$43pcKVD} zF8$Zzjv0cO1$XppkKf%(HnfO{6#L=B@8G;1OP&1|jEu&8zLch2z4k)Mi2fLRuT=|& zsG(ckep0FK?jf6rBKR$t)&Xj>y5ON z%O6kz8&XMVX~NfXLLTAMCuOf3!}pg|M*Fbu2gSNxs3X@kC!9Y)^}P_<1yvvX#iI7B5Tj4W zPxn5hT&1M(BDGX{vCozdS&Xg|#%?+niP{OG+EtX?YE>TRnZ43*yC?BX9vvdb49X(u z2T&f%dGy%|jsNntS+n<#xmGk8_PqTg8|QixeBXQ1u(rof0DIsHH5`Yljxx;J@Y3}y z+7%I@dPD7_KI10Cle9|KpdAO~gt2<>3Dou~&-N)Z>n*w!Q}wx}maK!Y%1Tp}oPya#CWcc^ycA z4=nwi!0P_9Uj3VVr-_h-QTT%du~gaY+2ja)7&#PU&#%p zais@dNQMk+>kjk0qB?#XL%VD)ePz4vm-vN%Ts6Me=SRz4kxY9NlyAws}~=;BPzi6mU z*IqZPl9}Q91DYvi27=pE(Tbx>SmMMPs6|r4MZh`oAc#?wg8A=Ow5>33$C+sPlPV!|GQg7lPri<9YVAP1DIqf7;WStNc0RJ11ygnL&s?qijblEcf2a!un=< za5(>ICTyT {@QXcB}s3@r&8q{u_cx}UWag*(?-Lrp(mJ{i*3(cVF`M-|m{K0>y zNYh_Dluz>Oyw5RMWIire+P?&vR%Djq%W>i5df#37BW{5R$m9Rm4bE; zdS?7CXF9?~)J!l!ABb5ph%xCj+gUn?wf;d=Cw$53=KI;m|8St{V7@g3T#67c$hCj& z`lOl8=pE_nu{m~{?KiQpJ1Tj+Ut@X<$sYW`I<&h{lm!@=C?~gW%^4wbHswA8h)wH@ zEb03mtZ1d_@0&0!WX0((Rji%ggf}NH{Tl)hRHAi}%yal^iG|pDo}BKB*bPYpxe~|< zpZ}t^#M^gBBJk_Y@nk(!Ystr}4}E`z!W>==ngx8%Ao?-bKOv@HsXi^_opKx=+Aj2p zb*+Z4Ng4)c9Q+dWv30gw+Y8rkNA#e3r(n`Z*{^lJzYi z?U`1_-4L46s8NTLrn=h|_Tf3Z)kv~=&eCoHwV+ZOEd1a7?sq%yzU5kwattx`zs&7E zPyiVE$O&JKr4)c8;I5Pw$^iw0vShKQ^nl;~C6Z^tu+}?vm-!|9izDx*I}I>;^yrQv zAZ#iV%9Ulye_!PPpD)C73<280@kn`E2h3lJov@U%4BK1(w}ib#H|p`BO>4g#M@@0^ znfs>aO{kxdgN*z7u={zYyRoSA-dJpIyc%`>u60}0&+6aZ$mg4jTvyyTrN2IOr?oAy zqs`UqOIK?iS(^kmkE8pY22()P$<$hgA*9mbq>+yPKps1ep8tN$`jY?p3^A7;l4~91 zyua?_t-|_OZPrbWuC0v9mBw7_MV|Dso>(t9Wfqo8{oQ(=Vbjpi8yKVWv>BIeV%<__ z<<0*wuMD*TJ=?@u$B_XwVb7CvoVL8xq2dn|7hTN9FENne?<>60rIiG^JhtmR^1Dd8Si` zOJuseL0oVY3Be+`2z_z0Z@bf~{c0!=ft%phlvkpD)*FP&_Ug{B4yUZbWcy_P>w|br z{+j}6$kX`sTRGCNMjbx$<)3YpVe%P2Y5MJ6-*|V;T(8qhWIj#LHl@9xJAOC1!`Dhj z?Tn@TK$@$vagMp?TF?Z?&DE8w7w2@I`Ijvld) zmx@vi#6O*X4abk-k?$EcIqm0P>~yE)D1Wvs;?9rpf&AnsKN*h!!Z_i`T7Gur9VO)S zQ_~no!rs2a-l7{hX7WB|uyN*h`nzg({@_0S6HUJ%`^$Y23}Zg?;=eKSt4@1!v9;2H zrg5ka>Q(1c*CA2IX|eqv`^ME6HC|nl6eFtc>r-JVqw4j9QQBDG=nLYA>z=FSx})OO z(M&hf%h%58kF`1;Zf9KiluerCyL%3)O|tAivdAaro2#%%*u6WR>s!j++!06MA$P}p z_%7cymzVsxJQDBqvOINX-ON|iYMIW^E^kEJk#~MhvH2EFohFy%ge2_VtM?yP#B)_Q zt|jbS0UDyk=0h89Yr+nhP9g_5<%lp~`J)h0e*K?eRzXK%z0ZDC+p1MMvXH!0jIxXR9B`1|6_f4l3>U4So zp=O89*Fb#*l+mT$W$#YQeUn0xWl0EA=aXovZH3x+Xl0rc@QUoll)Wy>~5)Nx%iOkG~j{8kV3vBT^GYd(DDgJ>2e{^Hi@)Tq;f zY}?-2w;LJ1H*|;B1{QqIBI-ySQq6!zKAHa%5Ov}2Lowd{17y}QaPQ)|+UeDn>cLq^ z^FufO&6@bWqMl#M&7*^0cba@jcuwXd5vMrtqhK&7k~`in|LRx2+Ie0pSnl(8P42Uv z2)Mc8opZ=M?6z~13Nnr#14R8{piDzQD}(p!%{}kzWt+H{zJKjT&e};=7f1GeL09`| zPi@8D$9^2qm!5H=UHtD{wnK$!rr1A+OZxlWKJscoM*z3zt-&}1ze7|Iz4IB zztc}XnTPT06pppX4xc{Fa@1vMLU-DmgRPYgG>yaBFYmkSu{F!kV(UWRr3XCIpwH4Qtl(7p8Jq1c}@-@ z%T9R-YaBTWFF&WZ>`ptO$Dk11#B<#cd6^41vhEuFc`nSa(TVV(YR+}ygO zyvjyLL~yNpPJLO2l#$WERhfMVF4q(9wj<)!?QU)W^0E#)E#^b7)+Fucgxc4>_O(ts zSO>Mc`tqX}`T9LaKX4LGT6Ws?4c8L(t$6B4XoCX>l& zqID~IwKcIDS^fk?kudSKXVyZCv`LovZLPX&2=XFu6W#eIy36FMeX@=6S&i;+(yzm3 zUK>KDQ>Uxn^$F7CDsBFWOh5Ck^G#TQ{Ljqn6YmNU_8H%E*E+tj!9T}HjeI6fmc7nX z8rjC_SBJ@T(m%u3&k`w+l~)*ZA@j~Oy3_3bUFVz6vpn5ps^ftLuHG@W8=HNg9Wt)_ zx~;N}(l7ILRVUA+ZC>R2))pid>QrRaJJZiJ61AUjkm+v-8MimoVYBUxJx;k0DDTLz z{3GDzyhYj(JBzquz-EMBe=y97X|tj5ri1r<*8UcG=aD za+UQoj*OSO&uVDrmauO;v{{q?&WMyR+LaN`TrwYBt=0L=c=i=%!~jt+kL!NOvDV#o z;)1oy)vF`=lf3f^3CD2Ry;f`t>U^NdzuRwP!_?1f|2k|f*Hc!jr{Cmz{k%q9&rG{F z)bTbZe2Xmyd78)0kv#I&m_@?LQgb2KF8!USx4g%l>j_e-zqmX;Pwsj*J=f86h;jaF z&xCZWdujO1{FW|Vy7q&odD6>xLZ0jR-LT^?4e43SIqOgocK_Oqv7F1ilv6y<84!At z&$WNnzecwoqq=wMIj04(ec$`u_paSF&yxn^M1C;ViQOdJ^rZaX^etiE3eXTOHXqvX zxghLduv;*b7_tIV6!VLX2@?;pjLHXfxid^2sY&7Dp2Eo5MhO?dtHUHgus|b7IhOTfhOp`Am4f^8wyZVmrr?OGIG&WFDvOcx* zaXBZJgvodO^jp5GrycTtwv(GVPVC0lPT$mHrG|EH3H!!Fn>G3Gj8Io?YL1&n#<4l# z^oE_S)C+tpGI!G`(O!wmcFJ*<YEe#lRh8~dH`9NeVfQM6HWg4JI6zB z&o|`Pgzh%#hIH!Z%HX3}{`WLe#n6cKb;r{Nbs6e=X-TJUzkHUc^KG$hU|mhZZsRIX zkRj};Gl4b8*hSdQi`02DtiIxWZ-i`fgg$r0#ZBUF{YThq`R88Pbh-B00Z+Z$28_As zRY!(we&#McCDTe-@7TSvo6oN1opmKYlxp{!uE}M&v_*c+-P&p1OB;MxHS>{&d(ZRZ zs+qrV8P-);bZVqxmAfQZAJ6Dv^jr}(kS?mCwNF|*JrgcTle9Ap%CYCd6SA3q z3KaF_&rf(^`N;#MWv4?t;q0P0-mBeMAd%NBgE~qh6@$!^b1yC9CfnDs-|PpAi2OXu zI=JU24UmUt;(G2r%PB0VuXeEjsfYZOU)iKl_pdVY&$g2$h?{u|pKX(Acy}rn7RQC{ zLUO`@^p!P#D_@p9+dRwcx%5HWnYVEH%spgz)e&LOuYcQNm!39u&9dkR5LY~7!oBpf zOxaGt`_0*5&kuqjY8EDGXZ700Lq%>)sFcN zTDA#Th!I$Z739-v1mJibMRw)Nm20ndhBG}o$P@#+>W;xN>e(@{tL?bAw$D6_VeO%g zS!d(HaXioMN=yHqd+zCsaYq7`$B}l=DLX9s{E&NjS^UW(hDz}eV{othzyJHc-%%ix z$&NZ1>G_>^-nsTea%83t@CfT8aNrB@j#HX*@>LJYytw(1A-k6ORbR&i{kFU5eLuVD z6fGa~VK886_0$?&Rwy3{1v9hsc}PY3y=ZgiKBup6U4Qs(%vZs;l`b4_}Iw2D!oL zM2?9^J_?(4*3WqlnFoLA2%BqCjtAGQOUk}_%;a-^!h7z@FJ)$Tp2A2g^9A?(LAbm+ z0YeUWZO2?SFSA^kUgoC^S%)lR9VYYTC%mxPmY!v}7F!?MSHI9}>^kQTuvf6W7HSQk zlQR+{kdDgJ(-6yC4^BAYgigP447oq}!4EoX1A`Sk4#7@cj!~CvMv4(z^NcLy&mpze zN5(Yk0wb?eYEGyipw=&I4}W>Mg7wBxcJkg%x_PVYV24`y^IwWu4|9jzI8%3eT<)-o z?<4?{@11jSN{k`Tz4EwPzpOR6-pa!oDy%YT2Wu8%gZJF6&kPof*+{(g*oipnru3}; zPP946M+RvV>G%L1T`1?R`P1Ex<_>=#Uuwd>UBG(R684<~+8`CHny}~M&^Sv17d>01ybrQz_1b zX~t0w;aJKEK0<`xX2?iIo}aM7C9%#nBoL(U7%at{Fk=Nw5-*|aR%*Yra=DlBSW-*YqGN_AiSx-k45eQ{ec4hV~$BVq`9d^fWjVFtW=h7kskrwH= z%18WE64V9JGFDh)#q%2pQ*Q!ayGWOkr%o)wgn72FYu;x#LS7o`CT;n7rp(4YLd(db z+`gfp?H}qvznyn|pbab%`q6H+F^1GhOTI~R)xiP+?)k|(%jjNsilZW`yQ(q)dTI%mzZ{2(^i}>ZSK~)Jse}BDG411D z+9;#l0akrG*+`4D$V9SAn`;a2;0h~@@M(}#R{k{*b?xK1dkUE&{}en<{|cvU;v?h? zbmjtsgt5)Apv{y^TGCK1<&ze^n7bV`ORw}Uyu^*ZT^rY#$atQL+B~)=BT*ltaZWsm4-C6nP=uQ8OSf?NA9o# zqQl7-c@bV83zy~6=I;4x58=#r`H}&i@%LMJ_ZeP#AB;t^joaE8@7}@p+!|)B=d5FS zij83+oumK!=)rW-+L=$e=5_ZYKhJvU z%omxnTw97-Vf4WcM8-l?FK(co>q8y38@T}HqDmpB)DsBur-b_`haFAtk!CzX*X|W% z+%^9llkk*gmPSWyDG1WFkh@ZdSvqqEj8aV~6Fih|P9Gv1geWDP63XgH=@*_g)DA=v z8DHS!mkWh(6eiY7AMKz>HYqrQsu1-nAt_kpgL` zxB9d2a>xbiEa9zQJ_zF7U3nrS#wjwbEi3@)sok`>Hd8NoYG>tf4cnBh5ukpBWcOxU|(rS+YNKZsa(! zaA}7exB3nN)ZWU22w6yx)tsZ^8B2^Lc3rfQ`1(-WFhXbxPPvp{+ICefxQ1 zQJVFKcHn0Wc&&kPhP7T9j1%OKm6|179W9=9eKr>LJX05Cl^*AJF68Qz?ZPsiijMKc zuQwPs^6AZ4T$^N49vjy_6d(?fcJe~o)?w=}zn^fz2^~N4#JhXlE|1{#25o9=aQdz( z%*LWV?r%2b2|LRu`k;t$9Ho+XrolxzmpX$DYmasn?y@la#{^lH(r?ttvcJE5| z3!Abm<0Up{zCB;adO*FyPKGjqGPoMA=|>Lcx=O+K+i_ZQ);g!%v=jN`I}A?EcyCu& zdF6LhjCJ?cWosZ~pZ93(pqXdLwD8(qdm|@4bY+cknug3L6RmGaV$B&Rf4o1;SWNZ` zYa@cu6RcTewK>d4OpeJrclD4XBp}BCW1qByvsNKw)j%vQc2%2rsTQjDNaxZdfoZJGDipkgssA^is zhhf5If%zbh+&nUFIdeyRb2z_hZ|oZL>LNYm7N0TQNdtxrbK7(2Bl61QBpri;Icly; ziyq}Q8ex@>43f9GKvsA6qxr!fglh@=HbnPA{z8K`Na4RN2s?pfz*_+FUoa3r@^($- zaZ`}x?!M-IrmJ4{Gm9W$)2)oD=lW3Br-o;#RJdB$5v$Cv8^{A8B`)EZR#Id+mFXIU zwBtznLL5RW|EDG(h~nmN3%B;Bh#+755U(WSH4r;Tgb4anZlG>2AV{U>UqPi6Rhsfm zax0Eq4DF$;^$t5Slfq10a;(XpAtTd3f_wmkBdqJ)z)Y(Kp_<2`YG@NX$@;`XW-;OC znY@r+3SovvMk#7(kyaO|$p!_Bl9(SHLv->PScEb@Ahz0!LPp7=Ao7>5#X?(9EVY$! zlXEQzd$vhk&V;gAWQ3~>kEuIlwk^=BhTv3-{Pul9L1BLF=)(X+yuYe z5$gtJPx*6BWqtA<>N(R5^+@D6(Qh2Tny}|Mwm7RdXRJ6DsvHath_-vsPTpHLFDV+X z6ktXpCqocSMkM2s!3Hew;u#mh>5sI`Gwv8q@}t!2C+$w@kv=GcG|Y$G+0gd#(!Uge ztVhZ|X`~+oL^%+9B*Zh}j4x$kz{*n~wVccQ4DY#c=AS&(-8fb@ZR1`(>PR7DRFb~- z(uQCjr{ZDUXlwP*Z~Ce3J7@{}v(X*KZ|NH2WCWtd7?`JbD4MR;OMVCzIRfIETbXxv z`RY7+^UFB=l!2T~S=khBeb|^ViOdfW_B_pHO+gGO_t^*JCz+h`z`V`rbT{jg@n)^bJ|WNbGlPzCmbzX#GTvBc__C2e&gpYwMm+Nh*|m1c z&xs0SQ2%n~W$HL|593i>h5~VfWhgMtmBrOKGw(dZ9hBLINkTo$Eo05xWNe`KiYt9r z#yhWhsEbpI2)6PovwY|!)>ZS}yf+v5iL0FIXb$>Kr_=`4Ag?zv;u*u%dqxB4TD#39 z`C0qakvvL6y!7NPEIlbVl7yFkcR!LhDXUt-zKu~Y;x9yKgIsX7CSf;8q~)H%%VO8v z?)7gAYc7J_;16|~^SApZzl3z_w3UOXI0jnc{L0GWhs|UeC>1Oq307yYkh!KdCfSTyIvXDv} z;ZHw{vU)f+OTkHn!LcZcy@lT~Zi-`;m4HTClnMFFT@i$iwO)EPVfT?Of>s)-q)DHm zMTjGjsa#ly2zaNGZ}%lTkpHkPnvO>|=|HzM%*c{`|BrLZ#j9&L^o*Cgam? zDB@lF%aako&Z;uiJCO2Xp|%L93*|5C!7ydv;hd^qG3T!hS&Au3=0_3(^DRl8@sjlb zd0MP29O=jE&SGxypzNkHipVlZsiVG5!q1Rn5ve~ULqWF4a+ElJ=7}o{xjeJKl1x%g z`5Q}qgR+=UsrVzMoIY14WvmtZ`X1_emW!TyPI`GH-k8mCYxg=AQDNkx4b4OOYDd30 zV>^vnSr6@^EjWDu$|-O0#m!d}v~9lQ=NaO}QI0dcG~h@#Kj5QWT((C#J0ila5BfjH zKwfhf#Q4$Xb$CeGm=I6-vQLdw3RL!qHZmsCm_&x;i2&`8<1znh0R=`G`0Kaqry9Z| z>?udGzn)8Uw?j9Y;?&qw!oEI;58AtKC-UF?C0{AK+MY~IMU&D=iKRT7_hf?b($xOa zOfm(TZg>8|bpKX2>qfr9L$3L)Ox75Xrtq15_wUTRPJ2U8KJ5UA0fVddCfoC5ncs8} z%C&N49OT0I_1xI9)_QKfJLa2;HbtDyA&#|;Uyea*BL%)D?{=TbF3wt(vXSC%-OBYc z=ODwL_GgTfM*3&^jBWaf%+Zg=Th2rMsgE-c&#eud*N<{%lfQM2VX@W^sqkpI;5~!RvnAdR&Pvz@#9oj)?-7C zzS?d4H=gOroOYmR)X%}D5@!u|awPq8j@4lkHqYo~9JxSm;7{Iad(K?wjt4ov^a-7m zYYF>y0FC)VhBnB-xity9!A-fflbqzSo>=m>fI5cKJ%O`an+ppgA(9^u18EXq>dHcA zF(QC^n}9XCpJ$l7mxjeX1*AHuw?$DqI>wf5OiAzzSxTZ!skRe_glIGG6d{|h^8`1p`?83OXSBZBCd9fq;D}YR&t(5kH7i}oBdBg)7Hic0?iL;G!CS% zJi@27Tz_giN5r!45drNW?c8Api=pya*cn37-`L3aot4=f$Wcq)gjkLTDKe zj5|tS(>#Ojxdus!Ddn|-kn@Y2VzuX_%XKRvnFa+q$3Q~+GP0!82IKR4nu_ zIm2Zgxt6d$d(3Vu#3%c#0ouk`b@Y<#@ES(XrX83lQ0KWL~aC+Fw1) zVR@QcuH+<{XMUJt?isSxM;>{M-}77}kO^xw`H%)~Yqb+E>8wj>Y_y(}?b_0sl5>l3 z+Ip$p#)aK@{tN?2RyZ;&Wxu)#Pv`JkTGniBu1(1WZEkmTWBa$oro-G#qDz;`6PK>^ zG<{}1*f?15=2%6@BtE8WQkgmGsKOW&2Q$LW8kOsu)` zF(&8~PAr-u1Ms9YkttD#nW!;WWGv^EJ#0XtnJK2>82iN!>lFj z+YvP83mMuVh5wckb`x5hJgQ>=v+$ajnODL@w=h~1kPe_C#h z6T7Z$ENN)}~C`znuqH*d9r`XN8+n#MH! zNd_To`b|4!-z8eY{tUE7l6B*np`}(vYt-e_W@MzX1l?ser;G)2j$Bp7MCO}F*~MR3 zjE&yDA!CQvSQFk{a?Nzq!JMF!8@JXYb&|ezs_Wewq$3WP19|EUp(US^RFJI*ukfiu z*mcVtcJ)Wt^$}TNtf%3l-eIo^I}*aF0|P5#K7VbCuvk~})R6YK#xU5C#bi*fPvojL z&`zmKklSRlv_ZKLr;KBr)>q~R`JUy{x5yI`o_Uj<=A69ABJPP0R-Fq@b9m&+}|EB(IvE?wY^npNq|oJx6;={2EV^VjFd1Z!Y-e0sMzz~x$JZl>{QL#XRp!y08ibM*T3 zx5jYhPQ59YGP6!EB4dpu%SP^V*HNo{97~N##pGX5Ez4!cC#!5b#uD zDI`cUB_wy)Dd4P;+KfOTSaRWI-ADNC1X$4ZnK}?mnV&_MQ3M&IfJh^c4N-6qHRaLH z3=N z8=>5uA)aYekTy$b#-o6y;wJALYZMQ8vZ7|#?mX2&z7$67LrL}CXf6QSpOVer7^w-n zt1`+r1vO_peA9DAKv-@H(<*)sWS83;%tlL_B zq@|6T=2FfvawF?1j=1`VQpweeXe!IVTAm6^mZ^sMrarwni)(%t!dPo^wLzmdbk7N6 zF?X!DhD+mKZ!j;qot=T)(K8+?931(?4;dhXIqjP=rwIr^OV~FKIi5jU2&l1}bIxYcFBU zZytH^QFljW(1%`GA;Pt800O%#&xiyV=y< z>1VwpM;RfrFQui8l!xAv{aqWJm7fgMK55h?Bh5p``+QxnUegolM;A&XBNCovQcvZV zF9UPFE>p|k{H-nAYbSkT&eId>eCZJCNiOK~98=x>*<#C)GJ@PRu2R3qoo8Sy=7gb# z7{`h9HxFIY-&kO1(%<^sSb^;SZWzz?8d>K7sRNl?2)ps3jIMyNBhOr>W6Y#(QbRd1 zUiY&^j%~)U{LW6kDkxv4WjO0wu34F8J#W&k5w+684<~+8`&$*4SYe-@p|=cmKK<53rEs4kmJCvVeH}kz5vKiwZw!rb3?! zuZd;n5V4YXl5h9IA*ZS2XBo1t;3$kqm=I5$Hw1aqS8;11$?B~Q)7nTtAb3+Q?$2 zZ@YKc?L;ag;*7xNjxu7a9qJu+<3JncX$AL=a2sdJCmjZ!JTYKrgkgo#kB(O7;(!3F zZ%stqQ=BYb!XUDajVHM@K7=za^_k;Xz`E_~goV0O;&QAqc4QoLAa{lssq#cdTD&bx zlt=Z^e%cg_3&sU?FlJJ@MQ)Xst51h8_()fqQigMf$Ih!ZRYoTsvYyIfAv7PIAYciX zhB{I3jVt3(`=*?zp?)At&Tk5~w4Inp!Y7;U--XTdhc$S6iJ`Ag5~ z1Xr?2UA1Q_TbyThQx%tLu-du_$#JkP#MsK@h7*f)fF-D|>@5_WOwwbMLJGFW3n<$w)^v6e8Z z$;+J8@7Xutx4P(aM$F#Qs{P4ZGE#hU(UokoPUMH-D8FQhd8Dr0ZO{Z|R34}aI|Cwl zj5Oq&HlN54@yro&DivU3OPi8U=CV^TX@s<{SR*{E4a9lG-8yaE&F^9WS!3ShT5Jt5 zSHx$8cbdWcw==G-l|Ky^!kb^IpGaSOJ8hIFDWq#HRVLxm_-$@5OnO$oN~MjF2SJREj6r!YRLes=dDA2OHvWNsUcGu~&N1xKufTXTMs2R;dQA@It9ItPBkjuII|<;_ z7(y?9kf+lf((~NCxoOOrf4GVA8rM$9Wgd){+KeGA^}ig$^*tHkRG#+HmgJ2xFxaW9 z@~fZxv_XEqhuonjaO9!+Q1>b5OLHV;WlPw%AvEL*8QLHft1Tt$2DHV)B8x2KyI*PL zs$oD=0!)C6QwuYL-!uNM{7g{rEd6t$$lo)57B51OQl5%#E^z9a_ZBxwwYbvnE<K|nmPr0R$<>4&EIbV=bX@a&* z!me#nX|?$96DO5NV?tiiMd0)~XQ9h+r4Kw4wkGUpZI`yiH`~;>rI2#QZhovxSr9qn zB_1it<^Lid>M|E||M%u=%KIW8_eBz&T2`CWq)~3iXrBy$=s<$y9o-wTt zICVzM_%T{hl*})2(i!tkN0@*56zS3CxeLOeW9~p2>-2&CGo~1jD9lKeuwdMPbQ#4M zRycj7y*Xpl&H@t7P?qx}j|3y?%F9S8Z^WJ)MIy{!hN-NFv1YEC+uG5tf$>dg7mvZu zm{CUcY`(ZX}(I295wzJl#_(257|V1rp(W}_Xf|E4Up8-4diDnvu33c5Ew^u z4P~rL1(;08d2Bsks7j+K!l2yRo&ug}QNY!MBdb!5%iBD+cGahM7#`JGI@-Y8bf3S? zwagPKk*9WcHFm5U^+_x1H-BxFgk2jVQnitn5s>WTtViUx_L4uD<+uEEjS+`#kuqOC zX?P&F>kn%xpZL<1o^-%((Eg0ax$cq^2%@<9PW+~RZL#sld6M9#UA1xc4QEW|7)crG zn%_s!&pC#XO>wz=brDj=Fe(erA=$Zcv>`dz1miQyP&>L;$`{5kPMJ~%HJ7cQ-7Y`C zs-D)?bjC7geacs7vaEC|hBMB5OlSR0U8RP2-SU?#A-lNT9O$hr$b}^AsiQRYtA5Nm z))Mw@2o3o{hBio{uqA|DSc{^2#4&exO_pry8Uzj!WYWpYV6XGbdy}MFaq2BxDl2|V zmr|MwSk|?kysj*J@>gf^rMDqu-ojHF5`~zup{&{{cMJi!L-Z}) zc4yO{)z1Qp_~d)(KD+o3@avM zg7jD~5uhxq_9MjXoU=}{ydwS-7j2E5tgx9VXI%(xt zO2V$)l$YY;s*SZ(=9vTt$wQp9gK#<46ULD=q^E2-K5N2G8BCIF5z7-9c3v!&=CJxJ zuXitv3**C__Wb~5GY-=5qV8F5XR@Y$OUsu z-4i*7vJLoUJ0xK@50bD`^0WPGP*QS%&ui@%d$J#p`&5eTrt@P2tBE?NF0}!U)t^&_ z>|66R9R)tarb45?*LTDm^hdfZyME0+LNt_xOTvy==gElL83&%5TTYH}=7qlG7y{CG zg@kitih0N{kz+4m9Hmnh3o zfOEc*)ED#=GCK2TghM*S zA|P5s(pJ6(mh z*5`U`j^&zVUgWq@9%Zpc(HZ#Bf0RMl)j8K-1|sEkijR?k{PNrw=g5QH={9HS19V6m z57rvtGOV%7IH8^B+sZHPTuZg3dwygW<33k^GpL66+R#apB<$w5x@CVRa(;mFw1j;d zq8_v_L}-H)44X^XbFdoF;#vS|^4bIw&+fKKnuOhCkY_4Nt~D~hTHqx-mqtD#6f8_h z{Iaful55r(`LZx6w=}X0>qA`*ag-k{fK66DAovhjy9L^b(vt7Vq%i4@WAP@OwUP2E zXSWhqLmCuw?U2Hf@FJ+iCuAuh`Mw|$fY4<6_1EkZNP{t9u}H$*1Y-wD)-Lr9yWKo} zp{;5QeiJBD%IJ7}D*YBF3Xe1>EDQu0-r`}Q13Q|8F@HjqMOU4W?c5n^0)lRF5mw(L z$+cn%c5!L~hBW04yZ)EI`WXXt88aOONtjfAGAy`Hf37Bql)Ic4#tZ`rfBj4m(q!L02Mt;o){byb=5b0a-oJw#65ghkpnDd%}d1Cy!)~7Wn37(TT6ak8bxj`vW zehRrVFpx1KshhFLuRgBs-8)BoDM@xEj01J2#LGk5xED{^@(Z^NNn`>g9x3%qz4Nq< zy1I75*b$a2rHE*|?6YpX&}EY;Zojgi8PTtA8Hcne#7&g?w zI%>{m+$PZyAH?l39q>?^IW~5$|bBuw2%(E_8pE4~ZCtsaO*{^T)Lv8>l zs|^6gl1z`RNFoU8*WITrHVxXa8#!0Cu|DC!h-RbQ#X_ zR3f7QSwi2X*YGzt)3639N6anju=+AIka6_V?6anR&3+_vTf)8#kpuRH3T=>rVRH#P z2=7YSXDw@5Br`ui7Lg!%QSu2a1HN0a-;mOj;X!<5Ru&5<0fH3Q`6l?8pcDtBj7!1ci#EiGs1xKPGAv0gsb0n>2g7LMJ{quTQKc{BCHl$M_Vc01etV&m$rHlqzJL} zEwY3-Wu5?{NGOkRh>H9v@dy>6BX3rCB%Cr}7eYBaQ*Vj|LLxn7kO$>I>+Og%;^w(s z7+1uPqGXYgzCIxA#0B@+X2uqwDZHI?d8h-yX@Mc2IsGP#XBHL$+&Ga2LTQ0AR*?wx zmA(Z-*~CF$<)vQQ4vC=9%2R(h3N0K`WULC8V}>DxVMQL=PaRlv5k_sJe;h+|965!w zg;qJ_X%Taz5@|8M5L#ox0wav@^>~3Ca~1{fc^Mb-*4|0x%@KrE`Hc-Wrv+H!6K}mD4g0( z8MR|K)ElziP0&x-mU&DK(N4n7kuCC9HsisS60Myr$i^Ut5U9I3UMoHMdPdH5>FICE zj_2AYl_qVH@sJAXu;5byI3zX+y?)WZ+AU8j)Z~HzNt=0&+p^033A=GFjxoiE(tVV3eR*yPq+5@FeVk>dI2sqd5!P);-r-Y(7#>CbTV>v&I#0N!Sy; z&EI-W0AnH@U4a26t-{OfZD+|GyCS^tVDMlCq~MFIEb1;y z#^Il7XcJfEgSzberK=4nXpRCRXc<1AQ=a4}UzS@$R({AromS@S>bZMy5ej}3FnPGz z84}O9lfQBan`M+9f9)YHb@WV{+BUse7GR;gc$AC6K$}M2h z@I31!PfDjWr6KRUQg9i7@}vcexVF+h;^(+@5AHJ^S8bg2S04BBR4?$Gf3~CF;%Pg6 z#+~|Q9fk2+y`-IKD!=sPt8T#GZ^*hr_Jz20_>2SYGklGFp2&K4Ls)4mXHD1{R5A+(%c&yhQ_&)K5f4Z|N_!r8w_B0&YC~X{;gmnW+dvqaog9aZe#SUQrZ775PeP9j zBCcd*wn5z{y&>g?T|i^SZn3pYTc>;>cadX;QY2K{lAq+R(8z4 z_L+6mn(B2O>kDNaNiqNE9^@k#nB!Ra+K!>m<3nz|yhC5}^#u*>9JGm+k zbx=0HJ#)3@F$_4Ku8jPZSv=)rFm)15zUnCr@+{j;*i1{^7{r}EaIYCS{-Pab<5)Ra{#g-x4vK!=vb z<58X(LuiEsr(pCaeZ!i>h+|X2m@-by7jr}y2K$tQDW{QN^MW(Zl_z)D!5rdOgTFY& z9LLxoJ^D5SfHf7FPJIZGH4c?gIi%?%9HW}>HY3b0&U%xt$T$g?WmgY*n`72i1}!pI zI{X>PIO97F=nRbFa^`M+B|ht3%RU=u=AQg)9;ur?%KmEVS91lnOTyl#p0z-XMIZZ`bc__)qO-{;sp=v@? zKIxwm*8-emP}^8ch0VIMdgjiaqeJbj2ft{3hUYQT1E+T!=lZ{iRXE zGiA##3BQ#)@1+G4CGn*1%Fpverkm-guP|Aj?t67`%`%9~PrS?{)ACyw^>7t0pF#c> zE`NhI$b0_M7hn5=ywx?^TKsIcEUR#SE0^(_<<2nDRn{z*bSOKHb>&!Ryz)%i!VBYf zH~32{e`mh#`OCk4o_T|^5MQ3k z-bC1uL1dewRNEy+z%7jCvE8E#OWB}+I%#1C9;tUvsg*`e{t>KvACPe+^RTGfv5*H6 z=R|>zK_V|4#h2ATzZc-@om5xx9C!8Gy}4~HaZW-gE18=+=UIMbaNIr#I|V}?%Aow< z#7QbYkn)Typ_nUczW0Z)=dJ>!3Q>1*fD-3<%6(vr@XoOIqWH;69nz?1M*_Lx>Tv=2 zdq*Jb^}5%D&D~*Fe`6QH(w5Rn1@O6mabR3Hk&<)Q$(}q0y|KtLQ@D+Hj*&CxNc|ka z(^lC|^_b`l!V0I1)~{T1$rdujl?>v~04Dteznr(mn)oTtQoc&xZ_l}0Pvw{WnpgQ` zzS(A;i7PL0GaqS_QNjzWZ2ZaPx_*hQqpS4OFYD#XPkB?W3NKAE$a7FH@iHu=jFztO zS>8HK#s~Ef&Nah9);IIWHug+;GF*$T18v-m>?`XE^6B)4U14jObs#stoPJ<@k&Y9S zX=L`TB1SqonvWPFqE1~hrqLhCf5e|`K>o>b>jRy$CabREYapLy17Q?`kU)0KUejVhf}X~1kRkux!m2a<}R3HEn(k=(2y@= zXoIaq*u}H!o0GE%6bOoC4*Xm!yVKiJNK@J-zoTFI&JSeWO;W!d%_i6=0~xm#s@=#u za&gQ!31M<^&*#8DpCjx9jQj~L*7{UBQZUr%XI#(t`JDoh08ed%y3!1{9V^8$5T`K z(ETjy;wo+(wl`$lZb(-;=0Fo+PX(E?%WXjwCSg81j>qW-N^HV!eIUL$?sqEw40Vo) zN&{h+Pwg@dN|ku&{LVBeCgPJPSzh@`OBnfP*|TmLKJW8Lyd#4Y1aY-johOHQ=jnlz z-`z-gP64x%!F3-4>^^a1ccW=Hp3%W`%8MiM+CqQjc+Gq8?1f0!g)^_r0Y|tq{q^Ou zwUKR2!B-~bXPl|W*v6nferp%b_+`AZyT>onQ#a|WceX=4CVGQ%xmqK%SIU!&t1Z*d zJ^#FRhf7%p2!r3Acc+zpnK!?DmiKj<>yS;dzD^ziuCCyJq{PU!6{itpjb^4ef0$v8!x1 z+T3zHJYAlKLfD-u;p)vEMpp5;`c#-xejMXALd>s87SuATChX*R?&5C?+*PDPe=h|%z$#vBZIs)Dl2j~E~n~u2Wy5lt!&UjD4E}bOo3<$Y%Pi|@# z?NGN*%2T?5HUPLlN5ACEi(1~*{YtjL|Igl?IPI3)bpf9{clITjeZ4bx_Q_;sk{AfG zDo`kkR1}RUJ7$w08nLWEDN3P1L?Wo5ky1lR6a<2ZB7#I9ERiLNY%!3XB`g7#KftQT z=X0L-``+K}>F0T$=iTnRbXR@roYVd5UoWS(Z+D+Q`pzRs*q=exVG#CXgPI2^sJ?sB zuxkL)utTr0sRdS}11g&BWW9=>FpwErR}}1n>KN!$c~mX~WrNimgj6eM;$%g{HXTMf z((x@r!XaD3t%z(Ya_+ggYD|33>RUkv>BOwAB7_Ec z>Qk$~`fv4ByY?Nfa$_gX2@4{-3T>F{pvt>%?ZfD62@d;VjEt_r-6*s z0-d$3r2HGGYFjdh4e1WFQ?888*f-|%*cp`1GsGwDlWRUfsBw+4C6#}x3W+a&G0H`V zx0Mg8CKjTNm;r zD^8y66jFbbo$MN@FgYNPYJb_wBZzyzPui(!AKAD^_cA@Pi=I69WUJ1+U+)fb?;8KT z_a8mpH_#I|@Y}M57liIl0U&t z?rom8@<$Q!OMKOfdMH{tU*uJr!65AEP<^*+7Xw%kG~jW7p%aj8{YJp#0h5((ohmT$ zO9#%sExsnuz*acAbl&c5SBxEkZy*(8(7?Yo1c|Y&-Jr`hM*}J03(rJ>l@tUBvGpyh z?+dM>*ySTV;+$0uZ32S`+Zr!??ip-YX*Xb_L-_Pr;c%cLcf?s-eG=pwLGdFSBBve9 z7H5+qCYgM$B=10)9oEI&1c|MLOfA2V=hH~oLD|)YTZyi_7!C{R&)i$dGB6eYe467u znB1vp@8tnIvUzVl6*KrW@HY|E${XP8&!k;G)H2Z`Wn@$AM;liCSDm%Z49K{8rxn$I zt1tJhu4Rb)PKbPs$2vM9vkBri>1)$Du)j$DDvO!?$t1G+mO7>%X${$GvLangR+eF> zcG#GmvAB?xa1(za&g&X|UDCj~XBRN_Ai1v|UIzF43Aybj#=b z0mmeHR@k+VnSjin8mckjnS)zhPr?d2qJH@nTEc$WnMtnobTlC&Pa|p`q$2EQ?+7Bd-sXz{*$iNe%zY=ycd}*|#`=AKBXY%scRl}RGb_#PD z+VB#mWoYI1iW-+zf9%tkRoNi2y}0${cZF12MkUZT&BU&siAlSRJI~ z&^ZI2tcdJntlsDI7=%3A3u6p+@>{nGvb{l7wpLc~FQ1uAh-?Crns!f#OIxn;d>RQm znbdugOTEeJgGADkJ{)`kCjKT|s!R~O$Or7av%ELkK@Erz2s=n3m$0l#*~L=F3%xvo zl#w39btjqVS?8UsgXqjPZD*B{CTQ*3_>eJ{iR@0Ox^_a(KlEgwCtK0VwEmOdLF+!S zKS=)S@3c4SQ2M8Q2d+LM?Ap~p`%A92`<@}B7-2O*+tyd(a${Xrbv$F>jD79w%JzAE zsq#!GeF+2d+}ev@y|2%Sn{VSY*`CjD>IyrZ^2GQlei(#3ZC3ml(_4jS5)MNLW1OA- zSeWULO{`|ufgNOwxW*rXev%0LnK@w(S-*&|pBaSx*r4V?D)e3md&oD!-s-QgD&&_5 z3U>Uo`u2+GOvQJqWWK=z@k?Xnxvu2#k7%iU(bFQ~EbH`w)?ajrR~l+?-^#9wHq69n z8G3&9f*Ke4m$@y=&&alZp(npvtfRYiug~iu4R=p(wQJq>g8F{pAi_9kXm8}3x)8#y zUfP;xf=|aB;*D6_T4!)!b((+a)cNYftuWikV^@t;MsAD&2|a-mV10_>-paS{&okL* z6~N@8!+orpvPW7%$b_$*1bX?W%P*0*JDuCAfw zrRUzW>!Hf+gr5J3nh!xwZXI3gpw(03T2W=zMK3KCJwHbsX$zpP0Q)xG?6_~cv!#uU zj@@@_1~2@pLpn;o97K-&*2vX1KWz;YqUykzoH*+{IOG;QCtTmN@hc45?%5eEfO{Ri zffo1$3^I*yXCR;+d2P7Xk)?OJvXd^(n#K%eZ`BfKU z?wfnY?{tzb+v9DkHz~lMys%1OV8h5WZU!Q@?bG4=brTfyrau6K7n2bt5VRlivVC_c z%g2GR?}XhDc31tT0b%qc?QLJXpJ@B@tb!Y;dbioN(aKgo73^xisBAij&|o^{9^!5i zsS@^8eR;|VkRu-`kMfTkw<;I^+{wCk)n23ztGa{u+YP+asz=+BzB=zaaUqkqfQgdm z*)0-wjl;SSzg0+l#b5qPS9XGNr(?Ha4{V0K3+-n86{(5Xp2>={3F3T0wGPsrtFP*6 z&xk`-m>HAE^#ICsp-*D;6X`egb<)=whv>zh{)^Ec)z4R3h*P%DS7U6@J3GtnAVz-Y zpuTmG_R%h*oGC*`uZ{B_*~1|02Mif`8d39LzX-bm>(He-8{fj}5K?cTn@)uP+~;|( zNZf>D^*m{GmHs*~K?-(z=-j&T<&UgYD#SsYtggwkR{tjGg;i1G=$+=?d-hyqcY}4} z(z;ha)xJe5w+cP~70HJls!a8B5ZENn%h1NZMbFPshq9QuPzifhymjJ8dquq&g>WFc-< zruK^_w|q|K?1SD4dkDMyO8?)>qafvkERv`^g(!qLCO^sXqg^1c{6|h1)~`n=yQ)9o z?FGtt`bU#?^1+pT(&?Ss;%jBkDj*`!o(jJyl7<%YKzOb_e?jtZ8OYE>r?~p(y}EBa zz`l+Va@`5>yHnf-E{1pDs$82u)t;Ml)9xdECQ`kF5Ow9w`|-}4lcKFRA#8iMg)lH_ zYqD1x?6Wbe*CF1{B{7Ljw|AxvsY@UX_0M}0CIW2&+GlCbOsNo_aD=a2Fs8R7&7rIM zO7S#NY}c4@5q5nydlT)-opcxn(<82S7jUJo5qS> zcmelR-U@m(hPDxE)&H&j# zYbM(eT`SOxdS)dW(J)Xj5MdA*D}}WUGyt)}ZOgbVdhEbfJ_9Mn3cADFoWAS3VB7Qj zPDhwvi(OVX&B!A%1nNhq^6%GU*o+PV}_#i0-(#R(YVq;M-prtQ2a0 zj9aX>u@|<9y1FX|!9ncP4p+TZ_p11iOZ>Uf<36N~;U0fgM=y-Tr9z%*LmjwfQbxPy zcclE)_vn4N(i^n8UDbO=+frrX4(x(ex=tJ9fwo;WAN?Fy5b~|It2(8BG`>|n^+(9CPoYf2rc8Nn3?l8YI__WgWnVUJ$EUj@ zdbJ7a1ij*_4x1>}SK8U*RCj$VcYUtiTlj_k(W&&Vc97YQZ|p5xpK$qhUYrqk?npBo zCIZw&^-JAj)J30=3Cp-zA0_P|j>Z5!0gz`VIK-KoafGzcr4OYa(yp2CFo7U$KATft z!24>;ZQfzZ^dRg9jC5Y74K)w;O4xxvh1EAI^?UIiRA{q=G{fco;xr);^VT8QSnbc!4yz%9A=V!9KH&8V4slbBiB(YB&ME+{nY0YV-{9b4?4{VdXx0dq^Bt$+W0`R{3k?R-xy=hc>N2Pd3;@|6!mP z|B9ZUqYmXwnN=PuVGpb%gs>~awhyYm1|e3Y4Gs)q7$h{`?;4DHe~cCON<09WcKW%x zufvahJ)M_-#8=*`Lu8R*g9!vV6O`(--9wClje!h%1FU?HPKUH=8>Cx$5yTj6B75$_ z%?b#Udv%&0C+=EBkfsbazzTv%K?51>TKvWs5R%g_8n`f44|`h%4Yo`!8k`weG2&J0 zM4nG0VJ9Q)RCzT>T_y{0t1@&%kFyO-?s%uEH)JwiFo;fBR4>x! zEQ3KW8D!DGT-{Mttk@}g%CiX}aw%VQ>!)nBRmasit70a)OmcBg{*zB1nQL_m>U-r* z7T5!MP@YX}Wrb2(qh2$}Qr)_%zG%|WQf zwB8?6x4G|x{4br+N3v6=lqp78SEtzPyIZ4A&Dh>qE%LJ(ZG%a1?sgjV^2~$?cSt># z2Ug@`^vT*J??8R7txr5j79e0q=#dtFQ{RS+9J%vJ3`+wvn0P)8PaE z>CkIs*c6G2%Bu2MZ8G4j;Y)u=yJv#|{xVQVrCf#@p0FVH8pu>!`IyR{xLNHNt~l8i zFP>?z$wo&y%;=^wY0}Ud*zm_b@s|JG$+2^#*F87+oZo#2-?y#|IsS#~N;e3He|9yk z;5li^{eFY)bRnq@46*j|-$hSyoPaE0sFY5Jr=5?RReKvs+424Y{im7_FX5jr#6;*p#r7 z({5^Gy^rjwI1D5$kiHE0MW75C4Z0&&<>S8vw@TQp@afM@@TL!~{{7ZXy;Zj5m1|oz zAa>m46QXU)Y4ikE?77R+v%XppW+QPtA41actiMz6jkdrr&C(imUtM%#U?Mf3aNS76OtNMw933>#cQO+1;%C0)$ zi(?Mgi#uamKL3z5MjP&+Jbk!u@{U}U*WNp15Atdd_5*~BJdLP%kb=4w!X9}IzO6gS z+k}yD6PB%HR;d_%_)i0$4q+XTPowWy{5@-E=(pOR1|s3Rrt#2_Xb4r}R64p=p@yH1 zDIKA3#ldW)ZKlWrdwQiYGk#VFHG15vZt0ZCCLc7GbPCV=pwr-qtcKMrq_Fr;-n)`< zz?AgxPcJ)e{Mr7S@U6sXEQKcwx->Y#;!phfO&XKmHT~`LeuD0G(er;&k@pd#%=BcB z3#36EQ3gXOty-DTc(#F=oef$_8StZowNvkwUkL|IxAhMHZ z^3Hq6DlLL#^^j@nT%PGZZG=5C{1`-MSH)o?Y4WK=zJ1N)Y}{6*8@Q`SDL?9-{y{&Q ze<#;>>f~if5D@3^$VmnSj*RAlD54(u{Z|EIJ^cDzMKx za0O}NSN=IrP`yrG2~!>!Lu4i2vpi+Toi5>em*SFDPFtx>dgeX2CQN?hx%WvIu&0Au zl|4<6ZW9>Fpb0iB!9Kl^Kgx_e(_fJBK9mo6=v~DqcieNRs>vnq#8v<3ohb9#r|4DQ zlr4VgXVjVOjzyfsk5RtJcvt!f6Tw!#)0f)qqD(Q$N)9@uPq@lRR_<#Xq;E|eq+hzk z)wkxt(C1{;TR-O$9qooV>QnXI(T5-_|BSdM&hkwC5vSS*#ux+huU(TKS2~4{qnI4= z4(L<%Q=cl*wxli(!hV2|k*5(g57J@om9T4kRBE%$8N~LG@XR3Ej>?}(6g|Srm0c=E z3;uLG0Xr2&#W8TK|1K70kU_0{(5M-}inp-%O`4XW7Y}f65bd`ZW&~~Xv7&CfnMP7& z_pPk>Sd}w4^evIuOpO#b1A1YQ^&T{=>E4tjo1DpKBp z)V-eUae*`_SIT_|yDhW6C1Z5{$|HM&^>oe#&S}qbaAY-RxNE@VMMiN@1?nKs1 zKV!g?lhlKB^m)#6G_4@enzEBTpV^}cGIOC8jPpEkk{Nx!z*qD*V8~f-F;sNgUqx#N}T!VZB%4Y-Wn2B=zs$D>2#0B)L#!K{@m_V?M18gludabz95yIpB zNSCxa$3wahbQ6q7mhpk`O#aCqaY_ECeO0ba3dn1^P1KoSl$W**+b#_b)YG0xn~5NV z(uAJy=#hWMB0j+pH}3}7^k)hFjLE!1d7&Mr(@Ljw@@HIP2Segm%gkwkHdwnUPt#u@ ziPdngBy*WjCQ>E?bw@es8Epijuf1T@%hWILTKt$0dh+U58%8C|JyRzFYF~`}w?@6s zWZouFclq7PNJGubx+T5`6J_a-m1kwOeTS)AFbMkrLq?uP)I7-GbuWY+6lfiY*=hu| zhGFK=ijOUOp40IgR9elk5~RYVQ%@tlY|zA>J6TX!R1_VK&Wm39vtkf}4#IXd(AIHi z(CDHgX<3EDp?yY*vqh&u*oe7&{k+fnyvfesB%N@DSwve9lMiMmvx?S2M=9^*g@zYl ziM>HYzB@EiX%^IWV9)J9S_6rU7>HEz5$H(F#-~wg^C{`yZ`3f>K~Kj?g!kQo{c+BN z5O(Fh5_U7-j2ZGu*wu4`&Kw$}KHKI;mv%s%rqcwPI)&KQNyO9vD+Ndac>`(207gfy ztwJVz^R1(|(x$!0H}lb@egav8c6HG})WB34)ITef+)bt#z*>1QnWv7j*FMr^V4T5^ z_S9ejtUv@-A`JAAjkFEwb_h+j=)0GG?W(k8V8*Z2G823mSmYaZVP-|ayU@N!gYR9D zB7--1T+3SOUkH2R)V4E+g|>gINFUI1gG~v$bdyW@)dyt4VmDY7w~C&eGAaL+Reg=P zgiNiYmrPc)lp*h)LFPhG^G;n&^eQ9zYFO4UxaUSj-XaG1HdWuF4ySI~xuR?;hjiPO z!d}0`EfZ!4y9p*`&bRL&{Vk*e;a2A5OU`eQPP#H4sgN#l6}~#3?fVArNN(!Bu|>^S zbxBpuMv5r~Y{WYIxocqSqr)VA3&p@0u}{aD|`UcG?c@ z7Gu{BJ8839q>Y>RAV0Oe-Uqun$rG3qkq_D~#4P2og_U6Aos^Yz>fCXMcL46CSAXF> z+CDB%jKh^P?;_i{5kc*H#p7uxs}% z>eyLhvPpU2Zal9o5srl;6Hg|XykliRf1MR}E9>H@Z_0PzCXO=UR|&hxC21i`w_R~` zW$bQzY;sC^ge6Q!8ZC5XOsgGHw@nnu3%bC)vaO8iI~ir)jv0Ak(vdFXa{1-nq*Cg9 zw%W_1LD-J~WaMc?&4c|S>>8gmxGGNup`c;T$;2u|{q|BPU_hXgQ;~F32%^qU=R$X_ zAfV!?KnC)@+0mfbcIw+72h-3iok?3wd+ZFNH5BxSw+csho30j#19|zR@e>Dwh=i4I z-gNTP_nOzdW~$5@QZgC%=*Z-afdhZ+_|tfj*ZAoWopfoYn;tXhX{a2&m6b91%P#3N zU@&-=xBU68h(C=cm_RY8f8{G*IZX^?7E~VE6@V;R1=O*7AM!j66SR3!pIao)Ho+$T zt85QFKPLsf^d1%T;@d+d>`q@+4%OqVu-l=c{Ib`XBDMyqh@`p%>RL{5SBKOS0~RF4 zU`MBKz)V)11Yz)Jg3)Snj8y@3LVIM}I-TmSPTne=!BOh4c-RGF!VDo-U&x9(9qg@E zIq1=VL>v)+lX&7~6)<&=UxPQsAj3)rSyypTA6>PfzDW!@H$bo&$B+8UPTOM8ZQ?-v zm&c5@QvS)K3{12YbkJ>;QC-)r8LU}JGuSf_VXv*M?>6uHC=mAA&aXnxFYIQ89rT;> zz>2Iq?CIVKy?E|~>R&lQhQ&wUpbF6CR`OicjgS?S9m<+;^*K3LfxCfs(xXqy_vGx^AqDE4daP_`9H3uT z&lqG}ok<_5A2irU4nnZWOEY7_OFzk8T{3A!w|Itp7|X~n#8jD4Uo!R)CzB4!Rkn^J zxar6ACEm61sD6PAd0_}knPQYz<;QqdU&$Dw zr+>?YAQ|zeS9+8|>E@ogscvX1#6?(1o3M@9r7d+z`sty=Ro;PnI@Otc+NKU?hY!w> zGy`LzA?=QF0m9ADJ&SJOdn+E2~B-+&XC`6M6Y1jylH-W;DD8 zJ67#%$u)Ul5Tdf1#Umq*zWFi(Xa>?`MF!snBl0hUE)Ax}DYK&7vVBVDZ@|OQ#SS!v zbQ-823OZ=J7&1T*7V@Am)evc1GhoqJWfH)J*sX))+a_4oZw;r1o}ZI~ zUV4uTdhzW+yP-_!n6tvJeK3G!l=)1E_0YCyfeEb)QmjbzWCG=09m_x_#N>8M^?XN(Arxy-3h&Tu0r)I&rQULw>CNLvdP#gw-fYBt|lGy z8Q!TrJD+mpog&)Gulk~XNqhQ)9uhD0yApOpRUc{P%rhV!!bJ{ZXY8QP8F+g}Lev{_ z{AatmFc@*qjvVA#y<^m=Om-VH7>DQswKG}S)2G^Q&KTS$t|s;j=G9LVbL>Ffv*SlT zS$Rh2*&EAb!jZ}MvS4LTpC00%ZLyfagp5c7eZ~&P3gi$Riwwp-@{%57D{fXJ$ulN6 z<#XOm-hrz!Y8OM=iAva&A$DLqV4~0D1^19_M990ROTHR^u{Ulp5g|Y2k#yA0eoUTP z<+R(!L_>C2$fxK7bt(6oAa(I5Vp)7EVQ)XT0xQ}v89Oe6&%SseFZr~g=0PgVeh~J^D%k0uYIwGRYRolMI=Z-v zL#?=G7B%b{2vyzDZv|c_5I?=N#)tpffg4=V%ZR5z zO>}B>r4yk>ATo$Xs?&&AP0+~kqf^%LA<#N>9i1&s8Q>dSJJDomB#Qa?L;K zHK-;PVvzF8f^ySPS&hiIZQ@x6X3`ZTZ95@(ye>BJ-^#Y=`8g@*rT3_y7vCO~@s#(v z!ftTN7=YTgrd%7Wt0Pb+>eOv@T>0gO;3F&{>`s0~$_*?{su_U0X19+4zBn6v7!2s( zkpU~>+)a{YBFjm&;$e_wWz1xlI^-K>^~frj9WUw+S$gckF{4kHcpBU=CMFRUt7aw# z4A2dynLdL&A{law468TlX5wIV8TrjbnRWp26<)}*I)Oavo21t} z^NeUCfBJT1pR9U7KVw&mzL$L_>=}dd>>`j~Ao3K$H<>&%*0Nogy|`uq#5g6S2}z-sG3GIWl3&O1V@CiZP0B-d zOlhx`)0B~Q%2N-uE*>@X;v2{-<8JRw`(x~jsJki;`a=^l#+dpVeMNRgsR!)&GoCbN zBcH=YjR7-O)NhcX%jXgmc2#lX7wpmivUaQp*F>+KW#rYTd>e0K+D-}+?U__i&TW?` zD;@mPle7sdlR4_A{*rF(Owy*WSGUB69upE|GOks&xzVd^YA4b@fpTr4QCntGU3$oC zSJEabgVM-8d8W)O-{NUZtvseaDxbY~tGvQdBX3HzCete?G@uwQlz3H#+|<_TGX zbdDz(@*o4N{UGd+f^8O8r$~2Z?m^{I0d&+lh1dr=0F{S~!H_z;Fckb`Ns>mx2*vU7^o(-lMx(uAvQ}H!0 zHF&ZbVTRsHlL3hpM626@m4>uM2C0m;LfQbyx5|#t7%*6^GWd2CcY|O%asa8%)oLUX zWwJoM6<;f22H?_ZVwXd{!0Y2hy&s)mCc5mc7%HNZ4x` z-3jDDKAAkpplx0B;#pDU($0f$^reZj{ziNdb`wllX%4F2fX=)xWnO=zZ=gqh80^-5 z$0U|AnKH0bACS7O{3wqh-pZ(nF~lz<-hf?s&WfWltLt6Ba2JQMb zgdybEa|pXhu*v?azPWD-7TH&c$3;hoA;i=^?WqK$9T zr%uQ#b**QVSNWqX8?Q0?36res5nVc%x^pTAr79o#pxEyQ#-EILmmNFmt@_XSY^1lN zC}m&2l8Kx;0i3j{8~RHI*-e@0LD|-qG3tJd_D?!%JyVb0_{PN{u$;z=il;%1l03C_pJh z>4bZPy)B&UAcF>tvQEQxFZwf}R&fk6HBvesCz|Szgl{IL9W` zymnf(f@nBArvWh#&=J$0&cU^Yu@e#}esyLVwS;FTP{rk*$qRWZKPq9@K=5w>pyR8n z2L>bJWLtamY9tLROg3Z|QbXxGS;lsyq&wSMh3TphLb^5Z+%v1qa2G$s-9X7COI^7{ zfDGu9-sFGo>-$)R@< zohk^xI!1e!Nkere#tJ~I*TB>Os5Mp>B9noV@I!!?AuCSRsAotA_qxrUS(XP(-O3uM z!v_5fVjdzd&st*;n0!FSG6-x9`(^Jmggp}+=~oUD>DP8b&oAtSu!H#EN%*wucEZzX8b;aOP@d0mF+vqFV{O20mgPWQ@JCX@}D5mx<1!h$w0D(Fyl zLfE6r;2g2z-|6D)^yBK9a6FroQ?_i05BV_pWU`7*t8nZGCapdUxf#Vh5)yHGCZG%-ldBtXUj zIR^*mlP+aTSlLDB{Wy1nvFjl_dQ1{%XN+frY4ReIq28x4jY%YghI_&XE5j*kxwd6! z6;cL|D%$u;U+RSXm1i;B<+prF|E4?`7t1$e-kg)+(^2w_GHG0BQaD7_`$z880ru*T z{?T*t9*m9IDZ9p$CS6_W6NZH+^;X*;E+%J`L-9A+lg~1GFh0)&ur?*ejslZg;$(u{ z_)}Y8rwM)fFL4!z=od$2l&Sq8f6}M^kQH|mcvi}lcWni3p>Lv>yR;@BD!FebPeRz` zbLwoadAE6=N0YEG1HN>{oaau@B6#uM`4(sA!ae)_#O}gaY#lg`gniwDK`yfUP1u2d zh1wO7%peqW#I~`}uK~BU#%!Ifp|**cf#px5)r&(KbpCXp+4>e@o^Wb->HJkvv&j5Q zYpZ)*Bwnr>QRzuOBGlPRCJj0&9ho#4B#0wf`J%y7$sLH1!AMpQbgCKH78H!!O!eDvnsOOtK68Nib@11>G}kWB+c7V7;n@OMz? zh1Y^yZD@}xdTBeVSU2w8i?XF0^$5Fm0(7S83mD|6%O4;5t8Z+)dcKHs$J z*Mz5BDQh9qfq}Kjb;dP)BqFHqrz?G-fxU7|mmMVPBzuGM5O$Mm8Kfii@{2)wL)f|J z7m;;^9U&E_`l!yc16$qYwR|#$$pVG==tmi8SJ%>i=3S{%?A;p!WSg+}WJQ}!lWE!} z_Qp~2i;?Iqb-uvwggO`)z7&2F%C6R;N2?! zCcN0GAL5~pGbt&4pv~9LLE3+Qm0?C&y?=EGu@zT+i#{)D(`K6R7QU;ApDbKyAMz=u zeq7oabti3#de4u3(j=1wI=eZv4Rp#Qc_1%LK=4!RfI2FEbTl9wcuJ=)D6Z zL?dV=!@!1|8PNQUN11JvkMFW5l_r#gCVBzZ)Fe=C<( z_=GKe898)uK536{>24NT&(P=cC-;}x+*;Nov ziVMiw9k9@X>kX>xTK_0|g+DsTCT6R2$yF7HJg~@RD z`rLfC56Ff(Ww&9rkIR>rzx?G>KaFtcx3xbe*W@t*Bu~9tbyqkU&!w(<2lCW=q_eKD ztG@{!+4gSft%O}$BF>CBSjl8>F~qn>d#SuxAjq%ZK>itPAbWNib10#a(}Eg1R-v%NalCpg!q0dCyK=&?mPh=WMB0 znWSsYu_x@vdT^23MTA|)upr7vc@I}ye#LaHB)L9U+1=pEvscZoyZXA>Raacyb+6&J zVNTCyE_80`WNYPVD^d$?s}Py!?fFZ%k+1IUGdYcEI^c>bpXb7!_T!f>SDHH zRv6gw17`ADcb#k|2I`?wNDO1}+uP!7Aln*)1BX;`SLda%H73lsTfsBQz)0gNBMw>7 zs%W>*GX@Sh^r<4J2^-{Ppr;)T@u}oKKw^*~^4^vA)V@=0^&9pK1k=~7iyH2x=;`aB z=NI;iu=5|X>z!KNQcf}f8|bg(leX72_dy*OfI+F%G37-$rdwFTto00pEe^RymO_YI zXycJUx+($ZSN|a_WK>;1Kttpr*9#xw7QOT!uO_vHD?KKwrGs20tRT=WmJh`iX`g70fP~t#$4t#_7aZng?;@{k6?kyggF2}Ec(zDnLP>cy!DhV6 zAn_*a#94fdIrWK&gR*58f%xX{E8=T6tGaKM+{A^6FcU~AXToD|r-I!tCimo-i4gYU zpE%@mHhrEtZ34<+S=c=nQ!8Zw?E3-x{#qSv(r z9qGWA$pqZVYJ7D#Ro;Eif9&XYJqW@c9ds(>I?SxVRETpLa+S-LErq}8fHdejF**%2 zvy#;!>9Y!L#f+Z}45Dv0;6D%-v(Fk`gLDl}CDh)D#?H1Mji0SQ(y2q%fJmc;QDbIs z?VW4zG;A7iMBYqz-JKx4RtM-YFvuY^8byt#Fp&hEVB)CJ(P&%U(4dh?11HWp35`rG zC(5@@!goOOP8#=`cLS#zweD1;jOz44*tKOjbk5|JRW0S(V9+VR^q{wUk(3!|*s zb%N{z9VP+oa*1DqleCLzzpbzfpD|-^HNhle21l+TJhhG+6ljaIEg1-?dj<>!5(Xsd z4r6jn91K7*xJH%?6c~9>Cnuy!yWyTOm@xp+W~XlufAv&;gv>EDzI77Uq#YC826oyo zpAX0{D~4%rOa#b3Kx)X#qux8^HVFFy$s0Pr`;^B9zutdK+*d(-l~<$<={E3I9>i05 zso_;5{(087D~m|6NgtEvc5c}Lq8?=uT-s|oT-iwj2se3D<(47-LrnDT%B3=CV$GnP zjIia8wCa0Y)nj#we`EtGRj%}r>I|LIWXxcKFL|b4G%1HvAlz9&Mo8toJXLSy5u<(a zUes$V#`W@stUWynAh$F-gXefK|c_*EDXf5xY_(JRl1p9vxN%8jzF%u54e&YjV}3RC$L_xMZN)OTZN zTlH}UaaBLGS?Y*5NQd;;UBgX&nFL}lt@TYv7me{F$@I&CPUf0hn@xzVM4kcV^{ zDfRXVzJxY_fq4aZ|}{!$XeE@M%!!mPRv(*?nLulZH%VNQZBZ()idq zM~D11LzuX_(xqeY4h%XB%*|{Ik3KRc9Qd`{L0CFvdJ=a!_L_GCryI5IRH#EKW4#r2 z^=`#bgOt18PS2;^rOI!AgCngRoyfue_)IH2_9(yay8^`dtG= z14V;TKpL~+guMA2O8Z`Qfp~hZ?;~j6lXs_2)ki5K$R0v$pq%uQOB-MHw)c>*tJ^YE zKl&4WoPJO~q_30j`a*HihgwnRH*wQPnglb@N9=9Ol_s!iudUE`8H8tb4AIq3s$U2= z9qi~ZNgn_7Td@--eW<*azv3#62L9>=IeoG|lwM(4?TZd+lnyc`6!ibdgwrSjCJTXcCZUV8{#U>;%CC%_&_<4cKVDh#6ue7Vd56htKXMi z`JoJuN4hd`XLVB?jRmDo9*SSaE_tu6CS8n8tonTH4JhMY2K^V27Jt~*2n z6MoXM%vgLvgvmOa1%YOi`RHiV#;EIcVtiBZn@?M`dnOIoyBa?;@~QP#VIn*e4CeXX5q?4!Tllc|V(i|BLjDh)mPG!6$%*cE6M$G}c!VWunF zz#(xnldY1F<3E)ox?Jhg(CIkpmiXv0J7z^fXPS;{zdC|6xD~Na2c_bs0&9qctD)ka zMv!}EmR-d&aixnNGW2N>b07Wk$CYe4RbgpZ*~d>CZ~n=V5mq`4`XF&gClVRpKl&<` z*$)gFnRVK+cz)N`G^6)Q90nUr+0v<}js-yqo4vjMkfVa6CN+JS;l*6*x6 z+pS`vkRO9!{sY879Yq4NtyTN(3byTX2V{&N?US~Sse`6$!2{uHr>yMT7RJI2CjD034CD>gvodJnF@{cU2fw{{ zs_htreStQA=*v~Rs_oTgd$$I($`)ei-Puy@*}GPr8_Kb{Ane0l&@iJc{6r8he%<(qfInD8+PWRk@W2;|vLD?2mE$Va4g8PI=_UbFIo(hA( zo$ZS`tVCh$y?_iN6%?z!uAa%aU>_X@rnWm7h_-{AdLMt;s%+LO69ep9f8^bhw~Axt zK5`W);0HY%9y$iA?&tI8bC1oS`h;zK3j^l%f7|bK{~eqa&OzM0SaQum;|&+Wpyuy zI3dhdrL1aci?lsv;PV}xRd?;8$%4AYHSMf6*6L{DfPC}kyILz?jCRbxhY3`@WkA#K z8!s#Zqx(lmheMr`N_R*|i7>#?DKU`l90(|l2uP|3$ms5v#25-l4n_!!9yRGa`=9gX zdEV~r?(cP9_a|*eQ$eB=A@ME#O%}#fQ+*NKHgPl9`85;z2SqNTCA)QMarv#~@YO#e zJDyo2F<0&Ch_GzYc6&E1MK7B zuJEt|2XPag=aNjoi~j(vP2wK&JBcjzR`?@{3;D3mC-Xu)7aR`HG8(IpP55Y=9i{`$ z!lWQ2-T3wcH8hiwY0ONYKt_0S1u^s1xaaH>0exhffi7BLA1D}GSaVh_tNUq~pTlPb3B!jE+6WUO$X5GbdJCz2u^ zmEt_Nk~wK@GBjtNdrG1#Y5Qk6*pMLBJ7CFiG0~=Xr%5({LhR2hE>Slo>Hp|6{M`Up zbP1l5^VHsHxG+a{4_bPs^9&k&FYA3@hAxMdY{PWd*EtOX-$4agdDHwUg|gt+4dGvo ztrskrvK(c}0RE~^)Ru+Eeso}*5at|s@4FMx zJSMr(epJCG3M6ZB^QrtQ4!FNl;w&=eSzKw&N}KimkhVi$AAYJ&W*Y(glo@o=*y*{_^ZRi(V(Ugz0BWn24t*VL3E?8;-1>S#^D zc(y!*eWg2TM!a=%Ub||$+~0((Su#uo2X`-k7i=qew*qfp=bQ}(mZAC-@U`$!A% zyt^Mlo#=NKqe@21H*;`v_*uI-Ul6Xc_JNbp=XOOA^=@ZfQRkQAeLfBo5ec3*?nY60 z+pOh$x!u%TurO7G*{=SrO~>NrVi0^6P95Fp+B_?6#ydGs?S>@|a`|;x;V8M!luSs9 z4^~5j6ibT}&FWNvUXC^%GdVcs$#ox!*x|m(A&XU}7$!Z@=cat2bPOw8j9dtgzN#+H z)2l{&8ochwqd%T&a~qPvDmyb_4)%pnQ-p|+i#&Z5K2sIuz*x(Tva~lGSCL`6#(lGo zTYAIg%5+*A^2}ai@{p8ZkFd()p=+E`5JKa}&`YgS-Q@gS99dQ0MNh6GrcjD7h~!f^hTLguoo-=W}|CI%@M7)!O=Nxl9)LRq`uG<)nA@X?1c^db|c8T)g@*^u_`Q9{Ok)9zl~*xDkhtqG!}!IIV# z=rk;QR#I?SP|-p4V#(q3^U1hQYgeNFQoh86tFr_747&(|Apr6#UygXiE?p&b_v17F z(QR3QBw!1Zv+3cmdP_p!$w)~pzHsPW!#?n8_vt@3$MYZ7_MT!UEhBGr6vv*>?CnO^ zzl!Qr>w6K5bzCH5SqBxqO{BQ&u%GAu!y13kFF0S`{^Z{iuPTy!T9l<{`1J1y$rJiL zTfhs-myI7R z!f2_-+_(Og`t6*HA(=|uons0taMWegRdqgpc_}W%ADkh4nLUc4%4sYMfc{T|#^0*?cSiH0qg;~`r#uEF;F~$9@vf3HAHI&QZJp6 z>X)x4f5gm*Sgt&%M(TvbKV=-Tl%>{f5x^EyA}1qEFs=^^nnpO#>^)jp@@`98ytvB` zuRTwQx0pVC*&yWRCAonJqW55bSiv4Q@&vDe`FQDgGfjF`8~&XHKy?A*`2N#oO9MU?!K?>Kpy;5`!{FaDMr|q~8vj@dN}E4?TP@ zT}O+fw_Eyw8QD`*=l)Rz!l(I#VS6=!>YDRI>#x0bemCna{gmqw_KN2?-#mCs)P($y z@#J5D8am5SQoNFPn(~`6dkd+F6;a*xtF+|jN@7mFpSOy*;xp7%@gM5^bZ^Y5!NRX$W~E_QpR3P5Kf%QQu@%95b-eJ+cZAS#Bo78yH2g1Mi!2djEM7Cs4t_=VV&r!_t}09PvL?P9 zL-G8@928PNda(J8`@Zxi*BX^lLNx5wW&8Lxy%4vVi)xQ{6?iowWVA8ba)6sz;d)qE zb8n4EU6ZCBWl~d(9?GwJ>>$`x`Rh@`MF0XA4#&AfH-yj)RItF=1SMm*wtk~LsFnDJ z$-gVc*51t@GPwps??{LU;hp&&U#|y*%>?KbF1i+i$PNcd!tOjsx!Q^gEzu~oZsqF$ zGGBe7Nt23lSIzdw0J!|)q8#k@#S73NeVeL$=xkXvucb%qo>_&X`!mlj_AI3@`01Tp zeAcPHSx{-_osUjOP^GRsxf%Zz->$Z;YsO|8V{h)>xTzCxVRN|9iphk`mI?SDJB}?la20 ze4Nf4jsDJymBIJWP;6AUwKs+LMhbz}D=otOdAhH#xt|Dl3!LlxMJr-aHG9V{r_hx( zXEpkBGIww|EtTedZsI}KMrxDII}&9U&(Sn?Y!a6M2HN`Cue-ib4C=a^()XweN5ir* z+JkUy^4xL%SwA&e9tom%F?Q5wI*8$x9#{@ls;nXVdxOI-qB*ND?H7kNoWxK6r7XGg? z^aJ=-AKmAfsQ@)(d1QOq`pF=@K}9jeNRoFysU!d8eA0{_p8u>VpfoAGA7mxAdsnTv z`)as5bndlhV}VCl5p1#r)7Wo$3k%(_F%63%#x@*_gl)YjHF0V?%2dqpKKsG6bgub1 z%_uCZvB<$n#lbiCed46jK0u$yl9M%YVcMqKZl)P7oGKOosXrDFMmeb^D#rn@bA2aDk zhiuYaWsYU4AsH$u0^8Vz9V&y)nBr6reJ z-(RxcfT7=#%K6>?f2Ja0*&!2Q993M;Ub+XY|e?O=<*?JEQn#ZU<_)SQI$urU=n?0i0 z8l~>5Llm5%yCzi7r>sLu86;cj)dOq=QyOnYM%~&LGGMyv{R@6lZ+@DQ`Mu@7RXCcr z+dy@X>ge|6_)Zj*t>T7SoV*cXRpY{RU%!2Q$H2v`#TU|^u8ZZf*uyaYoc1iRB&ZWA z-uRI`$Jwds|Ykv)?!#v5JJu4g=*tnEmk6_U7F! z$JB#{*mJVBd~=zrNs{`5oSE*M#D7ff$%L8=gw;E&8J+C zoba7b89O1_SB2*P-67^76{uG~zCU?MZAcv$nv1l~^kfKuJ%TlbR@Y{9-2GmMezepE zi-!2cr|hNzEFVrXC!F%)vba0XOpmTOS(0_+0m&k+?B!)KTog0h?7Ub=$ETV>bek2X zj%Ci4>p?~@oxM|jJCPoM9Q;)x$Q)w;a~sg@Ha|t1zR3uIg3L@(p99WEBY9_Z&=OkZ z12zJXl>k*P_s41uM^Z9B7XJ?FXn=B`&c1(hoIe!e{{Pkj?7p`0xS#>eSbmmGTpSla zFsY)%D{H?hzQjaiscG)Moyikfe^ms08NhBf6SzirI?Z^~-#$UMJ{JyOEzHf7?bbFP zX@HKvQ(3)I+>hT@yW``xXjmZTZg7t&XK%WmdrBi4rWHtjz? z3WQNh$xViGfASXv;lLGY2(E#0Ye6@DWhI-H8C$OvyMQc9Yw|A7?TOM&C1TqT908z1 zaHHPyGc2n0As+&d*xx~(ljbvfmgt+MHrCONKQL(EP-MtR{w#F!ZO?+0$*xM~)c~c! zLRZwNklz5&h|lt;E9TQ;vP1Aj_A52?e#h=eLJej-xkjLmZ!zBMKI*>(-&H*(m+D&& zp&8U&1={z+o`3mwUTX&*f2jFjaNWek_&ok*_Pq1VB4XiqqIq$~GGqa)%32mTZjz33 z1R>4WY`eE(AM#TAfE}W@s~0Ck6wj-Jr*^xzq=Uh=63g`MoARzPVKz2@9aMA2iKVy- zng-uT5)p}Sg)~=4D(}^AKW9mMdeE%LMRB0e>^a%JD%WK{XIGHv3R!{rHQ!fU^4U44 zlop#)Y@;W^P)&fH#_za@qLp=q;=lE#vWM+HlqNCQ*yrVeT%ECgA{eGf0KzVwjhB0R z=>yV$LeN>t{YSs?L)x9b+iIM1&Z&i0Z!>wWqE%k4WHS3Z zsCIl;s=IQcx39~}4O~-b`kzd0iBYG}?IOpochuw0#Ejag2q&Q;MKRyWLD}t&$KZ#2 zkpTg4n?>4P0>_p1YCo*r?HL1mZKp{X+1}91=h1Bn!C#x@;>u*L?i8%TtsVR2&2uv~ zZk*`$=^A-Wx?6|IT6*UX55RLwlEz9aB{y!?4eVFaXABg!g@@+gF}G~y`B+;wPgG0O zSJsH#!j|V!fyu%rX*6HIp_1;smj z{*G&QdSrVC7ccrF@v2pb3-yWjWI@og%-`_!AZ)(ldlwB?p`hgo-D@AfyN zXc3B{qHj~jN-Cn8N}JOQl0&3`^+(}rMi^z$_@UEP0EJliE+cy!O3W{|>mG(`XbW54 zIaU#-?{7`!GRTkR?j*)T=(aH0yYIdK8~iQ}^LWXfZ^XXKZAsiCsJ zQw!rWFQA0~zwF$D7wVmqc8i3meQhCxVLasGYZ(yQi?Z^ET7|5yqqy+;@Tx~VA*Z}~ z^vZ%C^UqWeewD%B=!&x{n))M`biO`4|JdL6Iw>LvCORd=rG?Y&vkLe^#1Uj z$7+#k$@S4m)R$|Yp<8@2hJ-0`R*B(iUaal(){k(o{3eG7Qsn)|BTgWuMh^uP!6&R2 zRe4noZnDCK!);a}zZ+kvD1`L$3-1oQSf}6hTp!joY!sRa`GSSQn~j!G*seyY`tNnd zt-6B7a;7j-{452*2;0-sqoSHyM=dK4O&J+uk%j}D<}C?i_Ujs<)gI2%wAHx$k?F|a zMFZX7`nxt7${1IUmdX-@pj(l{z|tMQ-Qp0h`fcZaLUdD(dW~a&V$GVRnJ+;Pr_amj z0*iIIqd%<^nR%B)>QL3l`+b$9SzbDLOJ$vJ%TGg4+(o&&vM;fh_!euS@WF@nSVGz- z&~Th~T*DbJ3OZf+P&3SdUo^cW;As2bKHW}Go)bu32=R63h;up!236N6fGah1uFnIG zHNzV4@Sg#d^F7j{{#Dex3g~v1A z_T?wSTjH4_u#+#_2p*^|xzT8)aElDny>mL-QS3Vk<)t2hww3%)Ubx;}TZ&f;j&xX6 zbnk6TY?^O8YXmmk>JVA(KWB=^r9Z#nby=muCP=4l5U97ih`++!TO;jS7sTF+UW++b z|IzI$bABCGxb;nAC<*v{L{fS4g5sP{A3qTCNw}StbGoE3-D5ysxX?Ii9|bizO`@>qxn3!5bYSppz?1>?b(Ps4*wrE50K0 z%-yni=U3Go+qF}uVlyhLb}@rX9Jo+XUMi32bS%vd|GszEXLQ6_0{`C2Fwb9=6Q}-{ z)w;LzfGzCDMsd69Ys>G2wAUF1zlsgQ-)685NoA~W{X{+tNfiIowtT%t&<-QSfH>BS zBa3>Ph12!$*7ar&a_P4z?${{qnbN|`BOK}Ir5Y0vs|`ZAWT@G8TyK%(5_#nC@3BYI zG{(+A3Bg@wb=`N9jM>k2`cTy#UTe~2+36BhCI=OR=swz}`mJ!Dhd^&v;ivSzJcZ=< zG8!ZHK8M6ENoM#KP3oB)jRpu%PfgDj=R7hhR!1)cHS9eDNHS-oj=25CW zmb0_#P7tW91&kE>y@|o)I4pOHNIk0KRiK0+B@qBxe9)&&CT!Kz>&)E8`-gWNB^d(2 zQTrAX=M;fpKw=Ki1)_;`K`N-r;J5PN!%jd!V${bY?UZn}xfm|2c}K(f6=vG`PCpyT z+>HmQC}X2>D={D7kaj;vJx&x0e>*=Y3=-zUjpqeS{co+lEL*#6P*vn*L8WANB+BTD zIufaI)sp5VUK9)0NU_n`mJXhgd^w$cADr7nWd3qyvXI^#=*>*y;aS4(a zWV*!#%NUObG;h0vr<8Fj>n}amjTyStj5yLXAbIimBm1gQT~&B%tl5vBb2Ljd8-GP( z7Fe9TVqO;-olFYF5R?b6eUyRh=Pvz5?Xev$WCZhYA+c9V7rlSXaHqBwnd^@Y#hF@( zc=vHsO2@w>5u>c8(Ne0prvnKX{Bga~Wqs-CySGD-Abv&Qr7lUf&B3BYqe^_9K}LEU zd3wjVzR+$Fw<~CkT}F`ni?>I@sezD;i-yzI3oDH3Z-t;8?)1T|dn->o78%mB#H8vr z%4s88hO=}NNw5Tv(EQwS)84@9+!U}@h{rALY;?$-hW_&&*9fQ4@V$+p_sr&QUu8 zUhP=zzWHJ;|L8>_2Y6`rP@W(5q5nuQu$&>#qj$CRm;le2lS|6)bOSh)O z_prpH^Hp8m1@F#_1}Q_dym-~beJozeJ&5mkKgQ_hPuVKUuL-QHq&i_K%eyts{$V_$u_O_Qw@r77wXmXi3<6Hjk(sF#c}MqGJ;-60~axo%{9dFm7U7 zf4~kvJCUf8;&%i5$=M&7W?JiFz?I?Gpr=lr~oZB-yJfDbi>Mo+xv4lO1039i&8twk@+r@cdF-5 zi1Oz~iSdM|XqSn1-{;S%`#Z%fyKHCnro0)s{Obv<(i0}St#mr%OZ{VyFP&bGI>g=O zo%z!`--b{<2O2g=AwNiGBdOWTQ?1X}%@^w=7AM0Sw@5RH@dmH4bal`{bs`85g#_3XvuXM*{x{w8lI>PxM?5%hOT_qD<8r#5qnpa2AN@&GR1{2|@NWak z03yirD5p6fmqMKjsjTS1-U3XeQO!v_zW#l&o_!w5y+~!`J-t2Y1Brd`2=%+1dt@;4 zn~FycgG_cFyeKwfGe;@qr${N@#^{fptI$I#+lXVhb*F;a-@!-R{G8&vv|uK`i>_~3 ziXP@JtYpyE6RUgu8aEEPUnrI<5a#(cI<`G2eQb#m=C;<;JZv9_)m&Ig;6(AzB!J*z zX2A6SXN<+v5YFuJs{&ASK@zMYn;*r8^AE^AGV+`>1tEj1QB5XRe!Tvu{7f4`h98N0 zcZPP;!0puI^Vi(D32F4<#rbP{=`8})nW8P=T1hdZH#2QY0~clunfL&a}?Hc z1Ih9$F>f%3MZ3Ol*;WIruLhLRin|K~v4*#RuzX2Kg#8NamIdgAd#lx|yWcUiiw6p8JE|!Jhwez+ZpW~CQ*0L+6&R(gatMi!VNh!Kc_k1wjeme}y^fsse zE%?lIPI%7Xch4nNJjf+@(cwEM=$|Qir-pt528gdq0lT9i+d&gjM+uslWZng_28*7n zND-}@OpJ#N0r41(z&zs?xhP|W(v81D4j(! zOit(0b{s-txOy8i$IA1gir>=jBDZmkgMhmL3v!AeV(|gqr(sc^X6F;Fm~l7WaxP8d zmL0iGm2~JAX48ve2J_ms)ZUo0dx6?Mv~dzzbhcU5D=*^QfRuPphpcooy*rEt4jD4t z`n33|toi?rwz(_Wq*S`hni&&&!ATO+#VF_~(hQeqc|{Qw0o$3p@29zd@A~+vkRYDm zcL$bH&kb=fv0KlURMLnRjS8}6lN=1iEj&a;;J7ZnKaNlN*_xW$z_qOwPlA1`V`?vZ zOl4k|t7)mqSU3KLx-{v}bHRv@QnD)ZQ>Nh(t!d2uC3`I61AGT7LtJ7n7DkAC$V0G= zQsdQ@HI$!CBE0ZJcNBcvT(@7eMY+C+}AmR_7>F{n5o( z;&JiigvU-sq%!Kki)RB;9tQ8SaW#(CT~yW_Zv2RL)OPJ*y)zU(#LYxkH8i7BBTU-& zEjmBc-!ZjAcW#fY!+M|?>RGC;a8Nnj*%@6m9mtwvVPq}t<$kyGV)yV9rDGe)_(3(_ zw2X14JmUzPngc5aSakZ)J;9m~N&d5oLbH59pw%$JTT8rd_dT!aIkKe=ZnMwZb~RmD z^*1A>&_ZK!*^jQEkS-nNxftt0xv)bG#}ad`+L6z>-xpXq*uJrT>Upo9a~+&^ zcY_o_`HA+IO(kkNiz`r)Ui`&8=Jh0MeK9fnT`QNmNjK#NFOHa)m@Kx5i`+NT+)K1( zikrR0G1iUg+s|K-6PX?W^EXNQiWs*Y%Jzilh7aa1UpsC02*5C|)X{|C`N(7y-IZ#x zn678zPT!N-Gev?1*~aSB(EqH@9f2;xk!`VoL3EWW+2GFqZD^Zovo5gEO$L8KQi}TW zf+tf|+s5G!-;hp|F!JYm_Vnr$5Y?UbmpPkPYG6|dwD&<}Vs5CNZus{kWtMu`5#Hm6 z?fQP2|266%N++5zD4EHra1x8q3gO_ApXJquAAJRA%v&X=1w;&tMl4jM{dW&cP0_n` zhYB?mvY-VVHdCA8IcMyCMi#-Q#5R|B0g-ACMV411>_gm1g~?9Nd)Q7Kk?vSSS2Xs z=VUX}$pMbIfFim8^(IqpI*yf>z4{!hC^R(Fk{_=pGo9QPeLAm(?I}(;bzHfx{!9*e zM0NC;WKhNG2A^fH|01aVQm5r=-27*y?LWE}lFP|7N-|Q2L`p^J6DjTcbnK2g5pG`xzm!uqe`F4CpzwW>a%B{L_ENHS zSqG2mvj%SJliF4bqQkTZ5^`tyzsaF{X@{RuH$I1W?0w_GTdaBWymek`Ym|lOZp12q z_O70Ye&52$FX|DwS$}pGKmLB=5M6GthWhtNX-d(f((>NIPeB}f?4J?=yuhc*dZs!> zeuq`ZD8RvU_lciFrbE>d*xJb)rh4S#_{?cWY%DaZo)FUW{r+ONM0*0zF9A$f4x+A& zzdz~48JLUN_5Jy)mW^0lqRj54yPd<*`$OL(Rz@|A%^A5)S)i}jr8yG4FRw;#uP6`_ zX8s!7Lli*S=81jD!(_5sNMITA>>Co!Q6jwH`;rR`;gcI>o*i3dK*1^UqX+_<uz&>+kZ2M@sXHCUVIER&fjfzdYKt1@Ka0 z0=?xjC9E@+(Lv6!EP2T4v~5Q3n4A4L(cyssiZS{ju7P=_IG@V4T3tEBkw1+CQD73Z zv@`cm;5J>~V@?C0l0J&`9?=&6VM-yV>fooiPf9dez22iEZ*Rme7j*ahsNA&EwLSmY z%hXW(UIQz_xdB7YDy6U1| zrAlKSQ>W&`nWqm6m(}S{?wr$X`FVcFZ2R3G;_D@9PYnhgM>c2{gr#m+gq3=HS0quI zK^l|8I;Mg_`;7lJPp1$-k`<53o8y_q8m<|rE@+>%Tafg(%|H>VK8vmR4q$%vhfR*9 zP85_O$TZ2c-Kd)~XUJRW|2}wa`TZ0L_bZx)zCPF?^`aI6naSd-12o3V%E8xXqf6C1 zu8rRi7^BLB{HfMZITq19WtYfAp zM!Bch43hHqvyg$nyqu&HbV=NIk(Bf4N88o z^GY~)ljq7t*W}`iMrLyoDQ!)&K(@_mR`Z(@4D|f2xl;V@JAt@VI=@lZASj)Ceubxw}DXo^qmiSCp|V#G^eG;Psv`_rAgM( z6s>pfvod{?^ekx)_o4X1-uMrmqiK%qyy^b&iA}=r3PRp^hWz6@CGeH*^Euc2{T!8Z#)y51aEy3Rwic(Wh4(0L4&O(h zet`Y9Z6&@N`crS^(awBGSd4h)t6BC8(_w1<+$UELiK})_6l}ge0Jh6dT4#(`zAS;< zGGCnQ{yI(cPCtZk`SIa_{^%_x@!7=}0hrT#N(3!mSJi051?fcdHXoJ`@nnS%N;w*Y zOGv00RJ@2rPXuj114a~ANO{B<4mYd4nX#U&Tc5PMJ*V!Mq565o)E1}1F498rtk{^w z;}&;A!;g@;!AbaX9jQO3P*oG;M}L z0>CJfIsG?=8-|iqaXrpoE>zk|Pz1uQ0h0j3BcGXdCp%sY{<|gT5G2;7g4)Tg(3+in zGp$t0ORLLxQwb}?)-Xf{o7=2}{dY*4fK={lw-e!dl6IdARzvp~hK~|__rE6tC;*Qs z7sn4+bs9M>a7dXeZ14mPw^Opyh>4YrqUxq}`lbby@8kFc3DyT6V9dhl6a~h$RrXy= zFhI_1lGslY%Q>lXOul?tk&NL_jFH&(pdFtGZ-A_17>o=4nk=P%2~CmI;>c`yQ9ZFf z;g1x6T-0y7NF_BjGWrL(Gv|ggMe@C$y9a&1|CrX-71B`3cU-iv?3DTJ9NQiNZ^3;h zW*;sl^7a@PSkf8Z54GpwFNQ9gUd3+dCT}(+$&$PXAJRPZ7lDrRE#-^Y@W=AdlX*Cd zzq5R#Wv|g_Phi8tvm1Vnn7_2rQ=gI0Mh|5*ESyhjUlNpP_!+^JfHI}pwu(LTEihn$ z2OlSu-SZuCK1TbVn5)RbtiNP*Bvbd^pJmIafZ;`2R#WHNP3MoK%Ws=O>So)Yn4uS; zuTnmspRHeuFXl9<=4(`heMnhSZ{C;;fM3j!;U7ZCy_JRczzseJ z9DQA1FEnsz2L%vP%lT{Zk@AS9p7w!-XTf*(jv#5(7foL4cIlN&uFb?=o-fy!51xj4 znTmN=w#nyqVukQtAC@RjeBH_xcnF-(<`x?po3y!Bm{gg#*@X6bkCeJ}{i5v3uT-}U z3s3{>d@*d6QVvnjGMvg;1l2KGiUBR zfH3r{1Ojq>@3pkQUQnHcUP_=-@>SkNna1B4vt_2DR;JEw7;W0YJaU^FJqd!pSjk9; zY}ic=frSNE$eQ~-m1YPaRwQ<4TT`&-AkYNeb^9DpxSx16^9ykR$nQ>G09&*e`K2<| zn%XsZgg&(*=2Do$b^`!RRVArIxJLbCvE1*EF+*i7#31W?TVyjmBYVN7B&lIohTm<7 z0?+cKoKyw;HZP)}W=aoxACr{HbsOAlSIWIMCQ%de?vE4uMw$iyhpWvcXc*atT|lX+ zgh)p{jf?Jg%ghaQ?tlvM9Lsj4gp`5tJp(?0|Q@)-C@U5)e zQ)nV`W8!RhsDS*~@ZJqEAsm=8JC$*Qrfie1mEDtbUe>P8v@YQe)dXKA1Y&|KpQuXj z3gWpNJn_jDr#OYw4DRd5CZ;Xbi>oY1j_OCocTI-!*vvYi!jLt5O{!!a*>m>cFK^6k zXKs0C=RDd$6>o?csTZ?O-g9O!`|k_HET|_aA_JC(BpkK69ttd`bRdX65<8MHNQw=3 z?q(b*Y`;5;*9?Str+M#oZVz{uUdQ6??Szgk^C$NCZ-DE^khwdK+WCToncgIvQJQ;t z(R@q30_Wp4=`>NMBMl6A^{cHT8BMLu5RZgvy!yL}Q1V_=x=e-ReiQbrX1p@ScxfZy zK8{>=#XVC_y`{?EH2Q@$wtiQ^fqiA>S$J#xHzofmg;VC)dpwkhl3X}=)qbjk2Izg%8FFNlI0?>tEE#pZ&Mg;tz!#r2$0Nb+~gLwtyCnEvYqzX zQp3rdC@e*fydC;6G0F(AF!yWIho13n`IAX6i6g?UCO_zqD|ZV>p=Pu*qz{XViqNH< z5-(GhWrGnfNA)^ADKqjA&GlGd&fcD(nyLx30J2lc3Q?IkS5xV_;ME5ej)&|)VK z-utnHG`SY$NJw3Z|8nu}%~ZhGM9|K>?mXX9F9{Y(nPe`)sMAW__^1i$yplkwG`-1X zqqa3A`xv!Z|2nCs>!h)IVcOF!WvQ5tTukFZs-XWft8F9y)tZ@+p^Ql(d*Nkx1Xy{< zapg`0Tj`Vf&z8-r@akiT2U4dkRAPC6R%SRE#-b=SMhp3)E9YDqZL$-J;8g@tu?fDo zG36RJ+pXvjuuVLXX_whL9D+$Q)~cSJX&@WL#&bq_(&N&+QnEozTJWa{)i-=Wk=Pvc zWP#4BYF%!i>%yB}LS#w$)0jV4!|{1oTM@OG0-|aZMUrhGJt_8XvAi@E@0JzYrNS+a za`x|O(6YA2XD^FfQ>^UWk>2w+$3lFrolrW8_w5@P5-GTtl;-u&eVfi~pp9J_Dmj(K zm=>74xnl=R_x0q>-#$kFzRQC4z5nSmn0Isn)mGqKvyXVxOB(;>w=Gqf^AUOi>G)>A z+gsK6O840Pb)K-zKZMq-4+(Mybo12(rL#Zqrq+J8j#n#*uX#^oIWu(zjmXSR07hOz zZmHrQ*qb9Qd-T*wzdRiV`}HeiuD1|iyl+v9A>3?el7}yb$wL6#V=5x*nv<`DAL8aS-3d|jbT6L2&?^(rXk zpu8Ey?fqlK{Cv%H!Q0bKMyikQqn|^zBFOWIZP4tn@!lKi826Z(xHC8pGKM*HEh`)p zfvRb9@v~THZXa-M7Da5*H(6XBr<~iv}+CSNn z_&YP=P-k$jk#F2>iZLxi82bG}g!=qroABccXzYyBxaMN>(TcE0i(~)GuE>*KK>@q3 z*R^=FRpGEXQPA;_TE>qds^G?el^50Ur}5vUK}_2jO@x5!`%tDaSm=-xH0>w{nYMw~ zxj>HTC39y8&ohx-#O#Fe-{2+9z(6{xMvn%Y*N7_4Au>lVlMqLvpK|u9=ppB?5bI6L z`kAG`X5SDI!lLNivMsV}m^SLLEB@JUe$0*UgiBOMVoPd++@DeY$KG|Z%92! z+^spr^7YD>tK2h_4ZVAt6a7Roruu!qGri7o6Nqo{tqJm9W`zGMY|qBASo^)~;4IFX zx?&kGseo6{{a3e=8tnO1M^^9T&;FwV)Yo9bJk#dKWwk4Xu-Aq8zdrBUFSsoEIa}3= z|Lja}nGOEdOfG?V!Sdz#3FymSly%JYp(amZymb3!<>}N2IMmBKDGWRa;@^k(T|f{c zl^dfhL5JNJsHqr*vX}*%7e9X*(6`RJ6+N=uA5y)i9y>|>10W?nZzMxz%Y&u~-Vd^| zVJd#iUQzV|JyEqrIj0Vh&vA@dXliSS(06sF_V*nJ3>~LR<+qvm7X6F_pUIF9-;o`7 zZ>vAmDDKH?W9V*JJP}Zo8C3?)l$C|jJhjnO?89u$AZkg+da@ElE&XohoOoB?KQO!2 z38r>0UJmI#4QVF3oI7neJVM&POC-ut$qAGIfWVb~3;tTL#~@?#HA-9Oxr@RL6|{kf z5g@vNn$48WUOJO&%g8B>Iq3Iqll%s~DM{}&?M^Y5enpVg6}Fy!m$Xio$4+nL(x;Kq zxqpr%Ke&oEu=~jTiz8tmeVOA4-z*vqXanGKvTF-6 zbVfbu5$$5sslfhbK&BKj{i8jgPiz}ZC<509zIe6RA?FQ8VQqPjzmhr4 zQeKK9xa9;b6bU+u&hXE%TJ@*ru+tY(D^;B2rrdBkl)AriHu+&!HZHM9ll`CYDr-)_ zF`-n-?K%56>t|in>8WRqn2?#%O@UB+ROmmUlp~%*xOh>;hg{hb57DDJm*`Pt6wi3| zRs0LutjbklElf3!spJ~bbz+-(b%J2<@vZ;KS)#^0_MnZh1i0^X00{aPN6~KL-Pd0A zNuXWJ)6jIbwSKrM@KoziJeo7Yq$)aWf7Ya>N+FC$P8`VLDz6gA$Q_ZuaX!DHv|CeS zT^!Zjp-u{4= z(viC9&9!??!>c)e)dknWURCJoni@cX_J^G_=@Tzy-HPD4W}Srp#JU)qoc}uh{iV2H z85Z}%)&L!s)Uj%^c@9%S2gH4hD)RHmEZlDGpxJsAfbReOOcwc8ihLxJ?JM0EWV?8a z=?xNSv9~Rhx;h{-o)ef#@p4N4K$ErojOvS1p^4(&%0Pa8&Fp3D|jN-9zg5&Xzkt!4?uFFIoXIXg+x!K+?Y-zQ6L>|GO z{i!Czbi6T_*?-sLKF-`$6TLPe4wEK)BeRdh2#?oUs4b%#m%tJ{x>dSij0 zXNme_kw5hX;XO^=eA%-AvU9!9y(+T0pk2UMYDB@1HjD$V11|7w0BC`Uexcn0p!TJ& z;_~x#=j`R_x>f<+^6UQRB~5w^SWA>bwj$%T4AlM({Rmh3>aMrc-@)q3{^ALoeIOA7;v{uqKBQI2H8HqLxIGtL^3^Q=1KLu;f zcLeMjE-Ix8!E}MtTZV;<;PL;u|Lmd+k{|`ER*nKpa^8QqI!#J6j#4XK$QhUVR6eH* zpE%6Fu*Q9#kmwwEM~Fr}FT4J+hwlyZjrPS}XXshDwou5qRC#$F|9T=DvwBZ1d~-OY z%r>>%xa{!FDuTe^?%UA&hkd!xg6zHCAL7T24@pBJ^KS@`0<7N!F?FYJ*Y$f^1J+o5A)jYH@zJmNTBw*ob@q%R)-`tn|J z;x8Sp=23zTV>_jso05gZ|!@NvOOI$6RV$6t>(kX+_J zL;N|B%QjFd^Tdu~{lLqa)sre)!ihzE_0%9CJC+swIT% z!6T4GW08}D0LlSjnw4-{6WB8XAT67A(xd$AjwieA3~-A)|35eSyYWF)P4Mx-)FDC( zbh*Cb2#YMVylM1AzVBmg*h0CYcv8b0(2duzAI9UbP^{b~q={$u8quMX5|oDovO}C0 zJeI(e6E-g(yO38*^gyylt-o!4Nd|3Ycx(+%t6fySGh0S28~gHN0Qbft&tAmAo0jhI zdyVWz95Kn{eaOuufw({+Pi-Uu|LNJ`=OalTdSdoFeiG26V6F4FJczZ{^hX}UfL^ca zvOVU?ijDWvl^0awvule5{<^e;&P(~zE0gA;baZ)2 zuCKDMhT_ST!cD7$|3h|9SLcdP;2f;Af$7S`x?U0;`#IH=uy_#b)t~++!C6sP-x1|4 zqI3V~Px2T?dC^b~ljXkf^WMqIzkIlhnu^c$)#1d2uS730ul{xOAN~m-ulB1hb8hd` zf`HG%+A?q0_&l$xVVuH{BsECA+#YjvV827DlJ}^7H=M`*C`cyvyJ`e#lQ>LuV8o<* z`0w-W;a%T7GSN-$|DO^au9%}q7 z?Q8eZdgcNzGZP$F)qfdGpJ+lkeH(rpDr|co;crdZmzLLzvH3Qokx^pykpr}~CEuPk zb!`Y*W^hggDzURQ|Ica|#ciZmlrA#LzIk$03ESLlt^Ba5722RyHDwh_gJ#-KeO^be z5TxHW*O2B?2r=%iv`m6b)zg;ij}qEo(?t1mkJc>vyAaKt$Q0;^Ka++h;;)F7^pvpe z33kh?GMv%Fx78rqB-!%AS*NiXGfzoX4dhu$EhUag;QFJs((H3JPxV&IcLwTOF*M4IR5>)414Q^`I-HFA#cSjALc@6 zyDHj5yeuZ)yV0|=~9N{DAdbA@0mrV;W5i8#d-T(25kj_B3l!^#ne=D$W{;LWewod+WdR=F>Wt2O*GrrU<&Eq&71*~>QDq%GqQDJ zs`1cQh1LX4_${g_FRyDrY8Tn0?MY(Mowi`9pEaWtU-;x7t%Tp3{H|P{_Idmk3n9IO z?2n9ZGS9UI5OnbIfT4KK+BEf$GrU}h4{psUw3IIwIn_t1uma~{?IK>qCUn#XSB)NT zy))QJWQyVH{EH0?-sP`pwCa*W>dRyeKJ-~MukPrrB|z4V;KF?5p5Q6{9E%A>`cC`7 z)wEtD-_NR6I{GC!k52nJ;gp8vJ?5i~(}j-8WAM(_{t(~_jTctztlT#dHiaMkj~~!h z%o(_0&t~%Xh_ByG>`FVblNb|=hYQMyEcEF1m){cIH+5WzQwWkUXOHP8Smn9??`aYx zoF*HC()eutuP(VKlrRcRCp&RWfv(NFu4e#MF~HN#ConmcjYzP5j=V_!0mwgRlBvMEYQ10dJ4gNn=y?Z>l6Nl2lDw{ljzBb44Er#VIkbim4K zDJ3}*Vz%W_NKQ#AY(+(eIS+G6&dNfL+st9kJ2=dSZTw!<=llJ9fA=4+KRouBd$0St zujBK&o)+anT05}e&ffjzjhHu`F2*kWf zd+&*rs+=s2K_bN(9NS@n-t?rjDM^kRgci03q|x=l)ly~e6Q~_n)B(&*aBdMAsVayl z%rPu|S=bOnuA6JLe01WwVTTcS8v?10bPqF0ymzddce z7CAovlmt)zbNF+OAo=5N#EY1y*00Z>4n6@$9xMbeH{Eu4GO0Vc346_$d3v`hr|j_U z_aAw>H;;e28NN)ox{@ZcQgN)Ak=uIWw;22`>6mwSntqU)t^2J3{7A&O(z#Js_r()y z5?QZ4#5$yHoHEDbwI4tR+9T*sN87z5I)@)u()-8FiG=@J;iJL$#)G(ShgRNocZbM- z&_tCNRi=&w>pIqkBSZ89hn(xbwgyg&_NoSjI;>zwOU0#2T32TKpO!A(rM~I+JInNV z>MB#O?q{tRWSi1>%f*Ji7{|;H-MgXuyz$?;ya$|>^i!vX8Wq5CxCGQjjp|aiRDCRt zFsWA*mh z{JCO(*wU{vo(g$uI?W;XDif^HjCI>MAb#mWj&qtThjE+OZ)=YWz$xfqkk<}Q; ziplHRO-M=Q3QA~Sj9kRWt|w8a5d9j4F{iv{@Vc?odDg+gfKNSPuN)nG@RbMCeEKh9 zqVnqhyE&=LIK%h>pfW;(22GtTw+i2fkcC-=P%&0a)M{Yr{L@;^LOF(Pp>xSe(Ll_) zg8KJxkK>5Y4!C1B?6neaV205eWGFDJ2?IIe3x^hlft>h2Y_whXm0cB9z(@|?v)6y5 zXsp?KV|frzw$MI-Ukg2N#JtJXSw~Laa_`ZP#><(uyZ*zIiY`GhFTP87ZanB?)9IBYYEPKzPi6Y5H94#9Mo$^wKKK;fj#h{pzcA;wmCK7M4O60e&VH0q z(T2`g#_Ap)_FAYev!jIckR(RJlY>5q~NvI!ipX3qlUAUeQ7v0HIThsZ(DOk0{;7Dg@Ed6AYze;bI_dN}H}%>`0sc>HuuY zrX9xY?=TKLj?Z!FpHOQ$tfjpU#6d^1?}eEgEXb6iACTD5j_l7{uya5eZQFdaV1AUw z@B0{M*&T#jf_0ly&HAB!#k{(SrZ(-jtsxSxH@|FYN{s3GJs(2AdU@Ko4X?6vj^DL! zC0iMyw9j4rGT*Dg57x3TwVV6~if7w$j2-FdE$SVCdygPXG1 z{;k2*UK8SK;*Bqa9;2rQ8!T<%Qskml!b8T}F3{}Pz1lI>o=VMg?utNS_d=8BRFr|+ zFOGdtNhy0G=S}DBP$g>1K+3z*dba9R=cW)Gpv4tS75f$OCGdn6$` z7#UT&vkPpH$1}rxJqqZE+JcbAIHbI9iqJP+bg#qBKJ@et05Nk+YWv}w3_|6cGtiuA z*5d!QvAb3Zm~Ff}T=s#gY*kVyO0TWN>}JD8t8v{(ZGN0FTC%SP8Tsf!T-0;V_UBK1 z*$dt-1Np_m@X$nLW6o8cfy-6>PhUQV+}=rBtpGD0G5CvZ@!e+K=1X!|((BA2NWTP= zsusEmeR*wQWE`{SEd}~xXR#6W;G0OSZ@I+iqmhE*@mr?Y+(ymvF*^H$-A2nZZ&nO7}ILB2Hj1pXUwh0?OeuVX9XKTIZsHCsS}O;7lxvO&bypj z=u0M?nF-bu7u(d7{gQm<>!}98rW#ey)4Kqjgn~%RfZZ&w>YfVk?6g#HKbVs@)*)jY z+oa+ydNNt8JZVI!x$&Ur=s_{kr6FZu$BHzs&Zvt^roH7{-SN*~YqC%&=U47?w@7I`|jYS1s#S*u+4aX`u=PPMkcE)D2PS=f*L z(0!N7{J$b!46YZI8f^5QyUbsw@rtvXUw^(<^uU4aQ@`}d-?Tyc%{-HX%I@ur8BdR< zYcV4s4?a`pQE6Fhf^E@w7UYb6=` z6~wu>V)usIA@9?1Z|t5}OXKp{Pf#Q%^IPQA=}%kLdAd&o3!N#!)@Gt1rIhhSC(^@DrJhb9#qE z$reXzq?fA}WJf?UZS^+cB_Rr|b?JvW1rKJ1Oqc8)OZyJoq=XN+<#iU#Ckj@F4^=&w#z)XG_J7%ZR5 ziHM_1qm(JSp_5_0px_#@>c)*dVXn)=&q^W2QWlTo!Nkt?M8qM(eFN#1CQUeX<6ryVTby!Ox+`GWV}GRw1~39r?1mkzFg9r#D15 zJ`4}>j+Nlt4O~lzl5m^*-)UI4TnRsdmdsGPRF=0}4QJO%sGbFb@Jf;!!*|jr!+*WH zj(7dyhjRlNwDgm2pl#(Byni<@=MHTGlQT3WUbkVwn)WYs%Q_qf?SYwGrF8+-a~Y?Z z0JSFgN0vq{M5+ojcEkv_q_BOk(q~C`{_+^%KdqkpsP!_`^mCrcxK~Ok`|6T~rCq3M z<@Vvs_nzRgbaKP{L2zF>S!e|d0J9n*(Rf@NeBsOc3V|7oWuJ_5z7N%L2YhcnF;RNk zzEb?clWy((q`vN@sM;-2U@(Xk_k;{rxyM$&Yt*?K<;$dA z9GN)p*C!7Pw0pC%OZUxmM77Dd+=n)#E2NHt#I{iLhF|0vFk}!`KFGRAG_lc zRRnI4-BYtGo#6gP6U6_PM@ z3EqPt39MzFG(UJa`^6jbB;^Dy!VePKXcV?=PMMxxVf5@Tmr*&1ooygjP^#*|6?ebGMuC4l5{lYpwMLX3!8;QcB`!Vx(&+#L0 zmUd+Q32^p$jIBwvZ}6>m>V?jDV9=e>kDKR8^n5BA|K5FdFDTr0^uaY=QQ{D!rO>@P zRKj4BBq$fKVW;&Djn@B; z8^qS|b5hx2kl_*Bvm;n{6{3r&Oa?++g0{z<=&e769UC-Ej9khONpa5bnpr5WU z8ny8 zW)<#_h>A40N5YJ$l_ixIId=4BaDTTB`-)~9x6)Lfb*w3-I)_WSJ>m-sfPmX`PWemi z3J*;P_VB)$CS^+*jcc}g4GEoc*KG~~LqJA$H_rsTS)H#YREw^7nN~Nv^qq08jj-D( zcFrFX5V^7}Z|*!T@!jdF_ouNVz8}cyhckg%*)c!}kIoN{{*}Oojm%E#ePny8XhTEFjjR`cydxuzrK16SNUR_lH!7liLIdiy?uOGeJ%rZ ziei)q>8{sC23|$pm8xxCQz_UEDqgvDAM#b>=1jGS(++^~ZrX1o>tz)&--R(-%9o0h zTZE#w!5_9D0!Ff`625iN1&nsj&#l@c^BI=F+-8Z#9yu$J#|Ef%WvjTN7~}P~vAipn zM}jX!TC3YDZBJ~*shG)X6JHX=7$ZhoKl28L*zt6$AI&o`?iS;Ny32Nk+r&`~T~ggo!Y%hWV%Db>^}S>y zG}(vZdYAF=J>?_&$h*bY-VJOG!*Ifk_@~XRguaoT!vW zsb6{ribTURX&BQiOxhQgDcW1y9?`>PT<}kfu(cE7uQVl zl6o$hX+R{FI_>Q@F9VPk^iP4hMR}j6Y^XlWdM2m}R2Uj3>^?oEw=h$oUR{#ue~6{t zN%?s`_NbVBs}3bYN&T0#4^h3V*di0B9*!O@OL4yZ(RFm@zm3j{V6ri8)k<=+WaUeI z(-ZjW)Xe$1!xz*UxBj_^pYgn{i3ruiRjFM&`_9!i4CA+dFSh#jW0r7Ds^5c|+_)Sl zTIrUXa_Lutbcd5bDvnDz1xnz<5tN z-U~tn1o=)=E+HOSXhuxwZW#_gvV+flG+L>ruDXLmm4$8|c44nerH#&laA z56Rd>eo58mzp1EmyX@fcvsd0LvA@ z7Hr!kmu}g2ThPIgZD5y5PLD%Vnz4s;r|j{=wp;Z_S#iTP zLOFfTrkUYpVl_Vc$wbQ*xRGn8`w|E09*}3Q#XhcHD57_-c$wL7KAAXgCktOs4nm9+ z1fD;2+-GLrwT_rKqKJ)e1F<1I=ghW#=J_hoFZei7cmKej(PFGb_|h}V`0qwFEW!HW zlcMX-ik|(di)dV8;XbF&bh%piCAu&JuZ^%FuN;>XvJ>?`C`TS@pG4r0f@ zE^HlRmVbQqW9Pl$=HM!CROaa1-tO!xx9&N1i5xf}qxq2Z>)Zhu7{zs_(06!rhiCDR#JIGoY)q(<0%cY(Z$VQLKJ!gW2Pi^ip4R*yEgH zDi;hB3kR>P_~IC}Pl~I=!x%NZtKk*FpOAYswT|I6TixhxH8rZrn}) zt6jG-kaYvJexm~0#G?^5Hc>m9pB@8I2J1j8gyTX}o8SDc^l09RN8LE}C#RYg$-;%b z4Y`92WFQYEW#M_OSKA&GmvBxFYWb*olDe;$&NCS#Ji~&Si;Qa)N&ZO)OZBkpA4nPP zm0v-;0gw@g&e*0e;1zA75UX9P`9egdkSWdXA*A1YJxmPvPH|?i1a-cUhk~ziH`i{+ zZ(w&=WF&6@v1RA!A2E=1AGDbcI=>zRT0090zD9C-_A1Zm_3QPp4xy80ai(_UKoeM| zL4)`F!V^b7t-AN!4m(D!Mmzbtbwtm7b>o^bTv_8i`ZrZI`$*3vy<+5QmE%h z6v-s(qmclOlSSsvjxGV+I-`jY?$qdj9SG=2tw2_hv_9ekAe?lfKpH4g5c*Xo^Ds~} zg>;2h*WvO7ql(e^+SM2$&T%Kf#N&=AGEdo0v2#!p?p>4S#iNWmM7m5%A}*TKYcHw_ zs!q-tZ;x2wk9?L&E<#;YVP6wA+ux=AF;!vRaRyC(JrCZTPcDkuZ+}a*&-p3#>_l0I z9CY6Zr@F5@7CqNF+o;5%(B$IfSt8@;0(Yhudx-2;s+Mh&sz(FY-)@?cqRqJX-Z_EZp}iO zwKVu$X{Sw|g;ssY`zCaO-vY>6{=vPr*y@Wuyo_(o-kT#rtE>c7n(1gIX#FG^Cu?*Ekh5Dqz=~zD1L1wh0*5lIVi!ZIWg7- zdM{zHnw^h){7|X0<%I#Lb`~igJ2n*NJfeOL3<%j z<5jm+6{yaHDTxDT+0~8E-pk;t*Kdz}nyA5$kzk8no!fRzkW~*cN7AAWo0;W%=_`M1 zeapjG^_bbGn=%4GarNzj zma#w*C4*DR(Rc_dJ79!}<8#MziqAV9X4Rc&Jb@APcI-BHK^Hwya~H6X-7E=BOGqZ- zlsd1Hw3ND?t!t)YiiR_w6ZuJro`Gd`NTFB-`XGeb7RNQiJC5Z`)n(LX;?GJ|Ydf2s z`M2}zuel+Or$u8`*1M;4`O2M2qt%Y3&cs1(XF^v`{&JApERoQhv_v!nsY>mXrCdRx zp_YSH@&)>AabG>0>EOU3`ABgi&*IpSZ9K{m@BX3P9dtDWz<3~!_5xH*2E_%NSJw)v z@Vtu`vbJx#)=tMW-M^|yvwhu;gQnz0L3bRq(F32OTADitcm95gs=)Fo7S#+~W%T zB%YtY1qx=hu(yonslLbj6Touj_hoq`1`i=^rpIjVh?duUMs6 zQX=+}9vk9JzALXSdq)=fzC&om4z$Vq@1z*4WaY-UUHOe1nv-ot%9V`#;f9TuY(h=; zVU%8L3N&aXaRkI#zB>REM+J0>ib}}=q>u_#R3x36jFWUGYpLgi(?)_*=JgMJ?OF{K z)*dN{5uuD%hY3*ZYoB-4&Djn)a!0^yHF;tv!FPZafo2Ht6`~ArFT-xyHG(m82N3Sh zeZL%2*`_MJkQa@-^0O!Jz3T7i=uZs6$Km6BuZLva^=)xO1WUFY>z7lYQn_&?1VK55 zSSQ+73~$CR3qE!n%LBPr!NmfOLn57kfG>0T(0ZKQ7#=OfCYF}&XL#C;2QH?dm9Y*N35cM z4xIU|?@pJ3e&WA#bX>3#N-b`737ELvrCd~6bq(7w8s_apz$AEWa4VT1LM zZ|8+y1E?0K;Y;wGDK~5q&g{g@`e=8J_>k1`un#y(>x$Dp;aH~>!^Fo#wZkII(_*!e z{^Z)_U^4X`t+q;;%cf1S=hHB;T{)rc{asfAsM8$QQC7lyGl4%L{Nyp}I*x~&f7D4=D zoz*7IFodmUgp)m|s2gZ&(>7YR>h&<`7w6my%p+??MIHkN%q3pQ5MA|6>x-|KSX(M- zpi6%=zKvk5hSxaL8~s&LUo8xp1y!2Xq7ozm$mz-hMzv22qy5*7)i;WvqK@C6mZiH) z9DD2~sf$kltLy$2-+*On0_^3rE`6p)#Qes!t~L1z5YpJY?p{z(vFXMG6xv<| z2x6&SV+Et&pE2$~yFs`Jf`Rc}Fs4?cH$p`+LK18sAP1kA9Ni2S^_iR5^QiY;@2U6@ zL7)Zrp&yPr#qXx@WBn)ynPbK{!oI}KjCyfn253kHqW|uQ>Pw6xTRGKJjD0%9KbLZF zzABGTAY~i}bRYt2M1I~Ad2vLGphO)=re7Li)bM&oUBievs_DYf&qu0TCGH?N#aO%5 zjMCtqLg+f4mrS6KY!?&S_J&4n2ndyG@VKx!)v04_xtc!DV~yng8Rv~y`5aTABgLOK ze1fVXB3HBXHK7!{isZ}h#$DJZlIB$2ww>hKW}FA37En9D&Na~hULFj#6JG`g&Brf$`_4$A5Zz3W&%xg#E9gHB2w{yv(&2 zQ4Kn;x|FGmc8+~Hmug$upJP6m8dyM2^=y4rFHR&?K%R+tI1hBKL4*P=i7v(Oor|&2 z8YMvI^M_Rveg>bG=kX^-c;tS`r~$nE8Go1I(UN{7QH*h;Zf5= zZ*a}@_*R*=ZDSQ=UtH&=O%`;kh!wLOJ(5;7MHj0Ny!3lIR`cZhY93!)-2WbJk?DwP zk4B5Jt7=rZssq3yS~>d8b|E^z?dD1P*K^APc+jIm;Q*GRVv4%S0{8-q7+>c6#VI3L z>l&54KE+nVwlD4Z>)&#?iN;fwH}ulx9H+&@*`1Zet5{z+5+jkqrChE?N z+O7ZzyG<)OxQ9|p%T1ksCuW^P$XMm?NOK~vd)1w2sruW5hPw5D9u4dj=b5f&1=!ez zXTfYzm|as+&&Otc_rC`#=M^I$BK7~iZ-7%*O7$(?prC6yGN3esq3fF%GbH)(Y->gC z%j@Kq=JDPoVx#i`mXFgG4Xvt`{3Zy&=o#y#$Bv-1u8NH;!GI^S2xRSm021w|;6K>3 zDAdWKb`Fr+J*6pgNyAzyGDA_25+D~Bs&K6fdiyEwbCzFSr=P_9YzNG--}@ih;PXgM z)_K++XF(vQvWhP*@qb_PyyuVtRs0$%J=A$B&E~k|mq&P##21l6Nw7{&`chT5BX-E& zTWXIh>5e?^xufK$ADFcSLk7#=f|KTM?Q>)|2TvVIRVF7we|D!oO@xd*-94&}BMO9^ zsPDGlkqD`(pL@*_TbU0-&2?H@+5J_G+e)P=3m3b64-jHBWUmyT~c>Hlw8Zn z_azCcK!-2gQJ*~WL}NCL9FDh`H8&9#Of9bKB)IwO)Gc1tD=C}qc+G~=>%yESQyMQl zcMTTRHCp=YrQSH`&C(f9h=a12KRMYQ$Tvou_BMLdqH17*{6bYU9qWfBK3ejS&q@38 z9-$=nCZUV95z|~iR0EFdz^BfFFy5tx!83YlkHgDa&jdFw7p?TMplG< zsu;(OsqJ(sr(98AEE82acKk)iH6D;N`oAk%bHW0DoO=EQd`pxQ?c!#9h(++n-PJeL z_&D%T=Z=t3KYXpiAGzHo*WyMBDsI)?aR5MOo%T%>?zvu_x+x@Qe=EVCkrOt1@CD-v=pJ_po>m2N(ml_X0B&R7; zcXKWks&|5zKg|h2)LiV#<#{ZP7bPG!qC#~5uM>g^LW6(R!(VgIys*c%lFuD=3a^mv z_(V%)A6Q7;;3F|O{yY>M6bNL~)VtCW3^@q}k{r20^3lviq|jbAG1zxn{wpirHn`hryOaO$ksten~7>6BA@=tmI_pIi1D-eT*#VAYOW$*J|l z5y7~SLp(mgHj>+}T|3N+7N+)p9oj$7d{Mna<5d&yzMd6?Ho998)ju5AtR5t)Jv!1V z+iC8>&n-G88P{uRd|Y>R5H#}8&$~eaR^FEoIqU#`pOWL?2c9g;R}Bn+%q(EVo1g2l zuN<{d2}(#267+fxs>=liAs|>xCz6y1R17m8bLIH)NCRoV2-eh9JxbMKiX^eXlx?RGbOMAIiaF4alnAC%)7{`>UAqm$ z6j%vhit@3W82}@j)*aJFc~EH?>o^%LXGKpisq?;NWA3^D6_$rY6e`avZfexxD-!sc zqR{DkvvL<@|C;+hPd(S>1VCpAo|vr2glxX>oiMp&7#A62WxIr>1>O|}{srfEdD~ga zAOLl`vppx$I@IGsyGA_3;$}9{N>krC9K5PiVmdYO%mxX|zz4VVoQ(&$I511|5#1FU zQ;W|JD7VlqGD=K2-|eh}iRddxzH(+qLw8`FmVWRFM&qJ4PaqH|Lhbzj4xLZ0?_Dgf z`+Qf!QZMwgFDZ7!Vb42ns;lq4ux@?b-Q^fc&3Jm4QysZ(FeC7HM}_C*t-KUN{WPwc zfP<}GOVi}mvHap$pt&`5WZ?eGWt&6y!*FV&0LU?AnEZ>QuW2V3g|U`B9I|ol>aeZ9(s5 zXirB z<4}gBUtdPXmO`xSKez6AGMqv#j5idotMJQQ8t(SE{=~I~yqEb`+X7q+TP1&g0=|w;Bk+Ya`iRjNO1)hR6GWQiT!;m;%{qig(TDGU!aa&@NlYCn5=@2 zX;6uZThnq)Wz(6k5{fvyD$Hgi{Il{_A<^Qy>*iZQ{f5<}wK*LOD!-YmlFP-?IZ2H( z7wVTJ*^5KQV4O>4@f)kp#=^MvA5}R=M)o@F{MD$}EHFC&%mi2?qEw3!d>!g0C-%HF z;HL_zmRj8}{y4Xn{jbo;Bp<4af8EMwp^RVc#V@Bv`h3;lEwi>Fx!CS>=+HX;T?x_| zHkhLK=6LEp_2sgZB6PqVtMQ5)zF~iuzdWZ?5c6zul)l?}LtKhp-5i$rlBvt9?0bXe9uu?gq`B$JbhW4oZzSpWEz^M{si`)*Sa8CKLRdU@X8w|BaSU2i)mf z!u^l0AuR6R6j}EESV`@!5FH?XW7r*&yFwZ zwu+Q}Ql-;^l3Ti;Tmv|e8b8&qtGb@<1}=j;2BJ)*s7ZJ zs5xwjbg>R=$!$7F$e^C>*BK50(mp>fRs*>W3MnK?IJWBL9kwUtW-Q}Goj)l*6Sw|C z&ocvec(!Ivx+cB&V|}zMaSw~@0IGGg2QB3e{*5sLUj6*v*KKf!<@MAkmgG;O8-xLr zx&Iv(?>pExuJYcQt+-(O3hz%c1aFFwBNztjR`T#iPPNIAmtXM(mS>Py5i4IN&5^EA z^%@L2r%pECP19R}n}=W%gRhMS6tpz}Wa{ z)nvu-&NUYn^(^)q8L_?X1&ydezj31yjls?{z6kbr*{^oH<0D~>b4cj@IXtS#8q6H@ zq?b7M$v>Wm^1QOYwLwXL$Z`pB`}dmo_rq}5-+TmBYRJtU<@jQ5@9JkyvUVCpZ0!BZ zaj_dFFJN;tc>kF&v^HJ2Cz&4C3#x2*XzOp%o&kGA3?Lo8B>@sNl~QVz46}jKU1|2G6;JaB+qr}5wN#h6DP4c5cnoc3u9u|i z3uAxRpqGHZ3&Atu05QL-e=3Sf{>FV_d04A(U6Us}*O(8u90y+yCvACu( z;rZuR+`hrwdIjWbbxrN&0-tS1NJp$nWOZXlPD0SoL?qCR{mrc#-)ti4`}PtvFncU1 zsk&9G+FOf$$X-w}uPO)+2Tf@N^*XoK6ErV5Jp27Cv3&hyTqr5)x z(nQ!J3*$Ppk2gckN!CHG(1)wV)d&bh;u8m^+k#hIzWVj!LdZ%i#|5e3@HcJ#)DlDJ z%vnK$e}5({9TDLF6i)AI?}d4jyNePx0MrSvZb~(5dq_k8rva4QVksYbPiN~_&mktG1csaaynGOlY>LobRf6&%XalKu-?ONFd9>a zS3Ld%fo5`R?dof*w(hDT%0oZ}+B$~YvRA{tIU+TwQtlS?=bO0r98MN0q=z~bF6Ua6 zldq$7HY(6Pk(Yxd=@^7DYKz05fnTo{C`;WWHPP#qK6~$nsqHdQU+9}z96h&b*x@VQ zO8#p^>gLx~ffQ*OP%_q%aBM3M@itZAMTi=KBy@^^V^oV+U9a2Uu;CYt?6tW%(3i+4 zEOlzqaN3jvK4ApnHMY@dgD`yU!ueR82lqBrMFSzJ zUHswC9x|kFv3>RW3Ehs%bC-ItEEWD%*X)OeL_X+hbt$LVxw;%IO|MNc;?aiKa!M&9 zpn#p2ir>+f!FEZ4-4ukz0EO*1B722`63kck*38_3kB2c}KjwB@^IA}mx8=jj?Tp!7 zUE8^4WxPhMXT__H4%l@giebTa$OS4Lz`rf%25(k3WH|pHPqb7sg4dUn) z8BQGV$vaul>WznS_w-X6w{}7@axDLRcaUib-U72#zbZA71@l) zdV~%-41X^nkL_y|t20qD7tB)#Ee0mZUr!TNl8=4-G70q_zzSb@QrbVQM(;Fi1w8L} zHM5P~CV~M|;-fYEW$3NVKvazbD7<8NK0oG4l2x-&dD-QSm1CTd1OW~jlRej#9UtRM zn*h;~Yf$Wn=RW<=5|6I5F!@$3(4dBUabm2|Z>PUhy7Gfp*nl5f=?7$~ zYF$Z&Kx37zuVY~tp_@iB-Zt@h#g99FPD0(7pE+a@L`=`0`q}cn_fiDD zSb%@w*)hNQy1xYmn_peSsOji)8WKeBi1h7|^;IRhgI@;cqGc8xxEXv_@(p1C zJ_?YV?5iA{%~oPwv~VnM&zxI81JK$YT~a$+H<}Zi`@ma2_l1uVLNQ)AW$ndVWqha6 zOS@;usXc1qrR?ImolHvF`X5acdNmwoP_ykp~sqY&J zONnpvEEmE>L$+J_zGf!R_f)mL$_Kn*7}PIUDX?HA*=Oos<85E9AFEC65{6bOp2IkL zJ(M#XLA`D=)3%0Ub4T-GWBEU}i6~BRh1OQC);rjw6nv7Zg9-mZMK>;@@Bb!SQ;l4a zLagkin^Gq*JL^rxoJGv`;%H`MSLPM`Z~m=n9$i8w`;W8ySXa_*_PxUCFYvNmDT{;jgsv2R{ zfDAZk@|$U+*vKC2cjNUNpb&k}<1jv^qrC0@2-IHU>cH zB79gUQL85hELdL8%7NPOrp?L%YI}h&Ol0ii$YP=z?qV1ox57s*pFjWauS!1v`x6xt z128*kMf6d>;pk1PHJE3CA=c~b_Pq$d7z!So$Q9~bqtC(2_S(~l?Wy0vh9gbMni>E(?(kbhEs>h zYFM!GfAz~Ez#&Ek|sj8GAHZu|7Cp&4e$R;4WEp(;+lzE1;gdbk()Y>t7 z*`EW4EK;~dx24$g@lmk#aM3HWbWxM^8<02k3=Fn*?L-5XmO(uuURebMa zs|-HAy(0GZVu$4U6F}fa9CgU4#e{aQ+`z*kls;;|c4a!2&iUc6{@)IH;1jHv{AKyW z{?moq4(~G!uammj{m~OWc~FK^MzBgC;ltE01l0IJ)-)f3(@bq(gfoDL>ja^DLO!j}!2)(i$3(4Kzi)cU0le*m1~9@mWu z{2G>{kvk7FIrj>-qwmp?PrZiSER(D(lRTI5;~eb{aulvk59c0ivccPqoZL17eYJ(x znrn%E$ghYCxhL*sY3BzN@zo2nfducUMPnvP!k2h%H%~{CPe5R^*ca36Dz@RzZTHVs zDPBLM;pGXwxb~?vx>SoUg&*8Y%ID-a9F5nBK{eZQ+a$9&n zgQjBpo!2YyI^klvlL@$-J=}6-zIZNZ$CIqjSr`2-OvCI9J>n&uhp)w~wNJ$~ok{bx z{z^B(J#;y;`a-aU%QU!`f9UX?#hHbrgS zdO)(N=Z;}`W3l70kJTYxjkeL`=kvFKtd)ev*s^N|&h2P0c?%uye5`RU2Efj>m2a$j zKp*f}Ki8y+rsWG=GQwyJPaTbKTJ0+&)>-3z@v-NN-D~KtBnyI;I@51Gb=p41zoR3^ z$4Q|ShrO*fhTXQVPZ;}+>>Ie{wxGM{S0OfgIjKs>U@B}>2K>G2wza#U4=_=c_{7Kh zcx-^J^qD3{rLk^D`^|alfBJR@Vs1mGE3SxiR<4#!YZ8YL6jvKH`F-GvXw5_ET` zE<4$ixHe692>bJ*otb<*6~)HMC5SF`2RaTq#AG*cU;`3?J3sv+=apI-MYMMn#e8Ko zcIV&OJLn69ZyO5hkpQ$`M$hm8VokGh@a=<{`wG;pHPFsl1%BfN{>@hMmnI$Skve>^ zWr%@b<)j0X@}b)!zgaenc!u-Yf9tc?&u2xKq5av0D=wS@R-+}6#h36+f4&R5b*}C6 z!kGPbg?ALdUVH?QRJGy)t3hw_H|&Zi=ybUq6oPC53m82G03fR2=T# z|1k}>l!GOo=Q$pjJkD?ux_{zriH9!?>mgs|pS&sn69ss!1d##g#ZSEqcA&;kc#151JlsVZ^9a0Gn zya-oXOflqkzOw6+t=-RJB6>X$z0_g&I;n!ZT}*!4u+t#iZ(LmDUtGSkC2$TC>wDtC zwR;ZFo-dvQ63CH1(M!Lv{u9fotl=l0Y)b(Cz!NQOI(f~Ad=@$pCGdGWk8~9AhA;hG z(eG$Q+UGaUfuO$B4_@2-Qdi`b4M^fRBjQ01+by+buR$$CvUprBVp}Z&?)V`y42X#W zkN)gQ1OmM%o|P5@Xx<8q`Sqy~=Zg^8uk+pJ=RnA-zIISJgr=0g6C6-|_R4lEYD1jB zv&Jw+g;t;nH#18Khf_+G6XU%)bz4UEe0{WTqZ7?QQq@{S1daC~B`6~uh>5olLOX0p zHd;QHWEOe+@U?spxjJ|n(|%&_+hYxaRSCQ44xYmBnVxT=#`QMKcgfKW%b(7Y3$5&? zuBAl=SG57PX33_gyWnr}kUE#aZZ2RKtq3BV>hd5{Wvc#H@Y*q7GH z?SX8sei1#hQQALYjE~bf?e*d=MUck)jHM+W-@Kid$p*qepfPAit!O~A%CCnq^wn)D;-+;3 zbiPZ=oXh>~Cf)}z%|Vq$UcOV<;hiEgS{zGu=`agmJ->5kEOjikRm@f}uvdp7VRnk& ziC1TJd*R;j;tQXNw?z(%hN|`VCVEr62PJkq%8iWRhU_VOh9dl<;+B*&)D+QgqRD<_ zj@(QtA%a23F6e#v0C73!Fu0MLA%4duY?Eb0AH;xcM3SP4z zBQ~{m?y(e=bEv9obfu>oaKy%YRmV84!W`6U_$YmJyX;|ufn>&zBNIo3ltIPCowwAv| z#&I|RvBwO!x1=$mcc!|!gT3|s-UCh^tZge+_GG<@*dcS12gUz#Bn>RiY9Cn>rkWcU z-!vEWdgO5Y0JyHpb4J#wNwI?1dCMlnkgev-sLhHkb%KP6*js3_`+amxj4b@5H$Duo zDXR%NH5b4cXum0l=qy@$U3y*$u|?`(W1zHQ8xNj&(qv8>G6{=PPy4V4qbd?$!0CVzkH6@HNOl}Flxk){nhoWdrgc8Mf6N2|8;n&95a4)-d#j6x=wVgCU>>&dk2%?)Sb zCFwD>++m+sYrUr-9jvXXh((#Kb#31@+j)l${mq}oH{XA1!vqZmHoDPRSUUgU%Mc~f z(f?!_-y<9y2nXcaQZ@D!Pom|O6Q6rdA1!8~<(M1;1!Uv4Dqwhlp0i;Ihs*@ga;#)y zc}|?7@BpTvDiEMRYnDpi#1#RLmG@aTmQ%w+hAQh10Sfwm61R~c#aGXJG}e8dYsmns zC5QCI8yj|h^S>XrJOj!YmH%mGV`uS0P=@q?P32O@j@0t@66tRA`zGl>!GS)K)g40g z|JpWIiZg5!)tqw_DLtyHuXg8W@ePF}xqi9yxyg#eOhmif`QqHs-Mq~d{LA1;ec11d zdXw=sllc|P#YnRYoRUeSZ={#mtQLGl$FqXEkHa7gyHhO$@bwLpxd%VhljA-bXQ!Nd z6d7VA%x+2Kw2+&zteLp<8s(K={}rm054Iz=mtiN?pZ*@dvtqxYoElQ>mHA=#IO?d2 zFy8?iDJk#$joT&*H(IsD1q1q0E*~yGYaPy`@;GA^Ia}Y|KD=pO6!v>C04($@77`T< z*A)tHQa`yL-Ce5K_z{db)VOX~CVj zO_?u-_q5!mj3*>1%76=i>%zPCyqiQ$4Cu0lm^{!lbr+S~4a;?yTKX2NUyJu!RsPlE zH@Q?(Z$En&5XI-V$i>=Ck`}>mjH~*q_DgII|%@_Y{ogu2srn#}(-!<}BNy5|=bpcAHcw-^D}y0|}?#~8Jj{#XBU;vA=M|JM!y z9noIWM&)c{bzrF(AqD*r+zjkEH6gS)X3WmWVPo8vwZAYa<=PRQV!rj;u-7-^BOAqA z#xUkiHDnG+as$MoqX_gKY1C%_S3_MbWwJ`ruL{2#!7R@}OE5O(Sj={P^wJ&HjyV#N z+q5B`>=EbTKR3NavqR)oKezSmf%@KdEGYJ6DUDazI$&aFCsn_!eMoG${bZf;hk2rA z$;b5b#7!b2&O;PZMBI$Gb@}E%a~oL2NM`oc6=IgrcV!lO5NU&X)0q7 zx~Sgh$BYKSsmTGOLjSq0pq2u>yZYV)Y$@S3Qg^}sMbEoCjA^|2wJ-x}vhEc58@NO8eQaTS7qJ)+% zbpRRSz=0i-(hs421Pbt%kCHq9^H@ZUJJ!la0$m?g4>b98YFL}|SA_dOlE(mA?XSf= z8y?9p;8$_=79yNAQP^<{6$OoONFBsx6b#8E|F>h)0iAiq`Y>KMpybGWnY$gAn~obW zT(HC?dGa?y-#_XH%2ew0SOVuMj*&(=TBuWvgQHEF2qP zeI)r`Jx)--ywma9ZxIT+v#J^1ZT86Ns*lx!7fC>~5mx9y#$n{fj3UOFjq- zJztL^UB=m!IwAhuw5}>q_Gp_;Nk1S$9vX1ix@D569E{Dtuz=A!+dS->$LkdDc6oA# z~B^X+T=W_vZhrYa?~45tnc8m(Y7;vWegnV=tn_QGlEa?}edM zdv4oQuN2oz*K5gBLz;!OyLS|;Zhd*p+8UytLq~v(I+;v#|3mi)#>S#6ZyC52&Onph z#fTtY$$mxLzxcF8{Y72zwc^z+$M=Al8#?iE>V17`?QBB)z;O(q&vK)^A8B|D>8@$o zDjqyQA)Xw&ed$}^3yqf9_wFSboGL7BLW=nr=Eza)D5vqM^Z*N02l zWIUHlP676s3bh(2)EADB>1p?hDMG-ALyC-DC27Qlor+zzJr(N96!R@{q6au>*SR-i ziONk6?OgXcFdf!O;c{@&A{3y>LJtc5w}(mFo=vnt6+^8;&mKytXBsbNrORHt{>zcD zhLddaMsl`B@K7FL%}3i|S8nps&t2Yp947jY_1`j^WaEZlkhEj7}>mnUrS3 z=uq{E&6S!RWC zbsKI+JhAT34z!*3G`M4Z=t77u>$zJc?V$A>qo*6w_Md)zP6vz4wJv?hoE~ zJoEb)z`T`O(g>SYgbQ0K{Sk^_>#n39FSz4W9q~lYGcJ$7&J5Qi;Y*5kPAN_j72}U4 zx1l|37nPbN;PkH{0<1@Aq};^@Z-69@S2hyH+gth?OV>r|-!VEK`6aq#{Jn7Ox5Z@{ zqX!C!uNHP`Axp@!Ijm8;y6XGtlkI(ybbVtWI*0LxQoMC-O4&uCu|LC*8*4HfYZZaK<%T}a@0iC_6rJ^0CWzXgf&9m% zmvFcG08(<7%Y$-Z|F8_J5z@*lU@_Fz##8*R(8XPSV|H5sPVmR*%;a z^TqE7XNZYpQ-Q+ALnCKj$H!S#Q0@pOx%>DYh4HDbUu+!s`|miYrg!?g0kA{9!xmfe zcP6{p9vEFKt>9(h8Chp?aN5s&jt072c~Rj@F+uw9+X+(kKCp@ANxwm?M_PXgJ`!lF zEW7!d(mymI1ls9$QrY&A~UTpVdy#7;~nO-&;da_ zt;&Rt6j-Du8qh}PYk)u<3{DXy<-zRP=OWS6cGmqn38O3Gt&7$dT#Vla?vR#(B76d2NSn`eggWHlaOzI+tf zNO4v>a0<6Nr5co%O6yY7lnZ|(z#7xNhxN^jHM;Zm`Qf5LKnvWZ6{jf9Xx|-(6i^vB ziP`33+Ru&*FaOZck9SFxv|gQUds%n%saHSNY;w;Lt8@ZM1 zvrSOUdz~al(t|XezTxErVMYdG(Yeh4&%R7fdCd5qVH^4*=mR?nSNtdLZRwLa(Tz&10OUm%#m|uLr~Z^Wu>TMAs^RFeNkVqU4bJ{k{6_WJTQ{acJjt%; z1y?T_+Ew-1aL$So)*xegMe|q9%$nU=z9BPzB68|w9k(p?9)CLqF+*AtwnSFIMm2NX z+H@`}kx7sgnwE&LN9kEQiLrnmrPL zmXpr@yQ)7MBONjp75muqeHgnf8-;r$W6*zZ-hmNkx35Nab+uI!>9e?yVYfO9EEmU} zRGg-0CY7$0hJ*kPZT_u2joY<>ra+U-Jt@QP84E>I8Y}2n*Xb|*|C;D3)$2QA!S6<; z&UX8YcXbp*ZgzC+j7XwLOun(1^V>4WIm(NSDqF1By;j1PHrZu7BkhVSX)urX zP2b!ZVN30;toYDVr8M2EMoEHpuAZhb^@=+0B>y_FG9EfH!e{q8xpVV4(&Uq81@OOI zs`r$v*|IbIm2E%%l#`0qSRZHEV`ok_!N*UOmUdp%X_!+|AG3}SA4`5(Fm^UHdsE^R zmiYK$W&@OOGV@C>*rm#5m^WVh*a8E3g3L5Gd{W3rmYaU(ow13ybq-*_R^q7{_!`huK6>Uvvqsc_43+yQ zpEm5F&3#JI=aAj)BbM>+IQ6%ow2Rg`4s|JMg*8)QGeh+D;OMJEW3DTh(2&`-RF%g6 z@EI9AzPr;Jh(-CCUs4uM1ZH(Z$A9 zqh1{inCp+}4&NE3U-+OQ@Z&Y&DWVkPMKg8UG@W2_H*>jzDIje^+JrP!Lq=JsI=RqK zYkONWd;!l6r*TXdy zdgZAAr`uI@O~xT}a2p~)oWEVn2qPy~H#fd-ZMSs7yZjB}FFq0dK3y^5FaO9X>l$z2 ztf~E5m*0gfH;aXBwTlt`&0I$c4gSQGv1D_qXZBV_I-D30gUi4p#jd0qe3mCKoPv;X zsy8wd8{W5aA>Sy=>9r#t$X7#SCdES5dLFI(DbB3)1Qs8HBp;vqBjzoLjXjg`BQ!3) zYyu7j3)Yt`h(X%?{Y?myDV1~SA_dn^$y`z*Ouh*ofXyX2&mh}d_xVON(CaAvXw4pHWsCMiSFYJmbF3xpy^f3=P2u^`PZOM(TBIv?cYGU4*Mrt9VmN7l@1=`| z$*Q@rQ|*xT6BoEgm1Mj(@L;*+xY?(2a(Z&)g}HVi_OJtSd%+S2SJXdB1{AEY?#vwZ-kgH}ZU+8b=|eJrmV73_ypxF_*h-1Im0 ztrRx_3`jVo@xm!Gt%#QJ59X9$u1rYKc76OU?mbtI%F&VAM(WBZHlc88lbF$I_z~%Q zNg+=kL3DoxzwNErGp?|P^iVN-lBtvJR-tpQ^E4qPyR1;=bJ$=WA@_d=WaPmg8h@y~ z4%viCe+w$-dsQY#f3!79937ISb-~7IE$fJbio8n` zef(ZR3&e<5Khju|ImqtGVwg#lCMpR(q%`~B=HtEy+MaTHE$KDGvEk0y)c+g0U6KJC zu9^y88x9b_cvH&+Yo29mxl_X9x7DD3M%0jtdg3ji_b$=iMjFUuXUdZ0KDOtzFeuXr z>vgWV^uz1N2QoCUViqzFBG?k(rIhQJOejV$H1W!3cMA_m)w@HKKXwmdOLA)5?2gZ2 znMB(_*nF#neNuI9v^{3C1U-1KLCb6X<)L!F!@I`Am}@iSir&?}-*RoJ^{36Laxe~} zf&2S^wJhNz@q=X)noSab?8!QCBDg!bDQV-YXv{wJX1osiBkBYrN(l zzU>+1-w^W#DUG2;3>R;c@%$H1*|{fruJY8r4!q=)?ydd_Ybdz+y5SRoh(G&!PL{CG z8})W&!QHCxLhVIZt!X4uTjsOx5h((!=~KnY^YY{vdXL;DYh$Mv;4EV)=|b?DX0O$F zRZq(82mSR*WcQI0)B(7N?tYgsl6uP@wtpKa(BG%WJhpn<-%qbA%S4@;6!SL|Qn940 zYNRZb-D5i)ppCy0^RBF-SdN^;XyPno)wQiw*WG9|Q|NW#3_Dp#XCe+@bThqhtVn}^ z@ls6pr_F9DF3f6lx9(d3U4PRf6(`ZdCrg$S5@NjTqn||IH8T4~Z}^m)^Icy=EEOuc zDzJSKf%&XJOitO3Q|Lz6x-f-on||0GiS6xx1lOkh@tfmjk2Dm8rs=PB9J-GxsJJwP zSWvk!7c6mN&-ugAqfVUYA`BKQ{--lQXa+Ip7}D`+{$i;#&rZ92qkxZeBb>Ncx^*QI z1-yhhIZv|R%5Bc#O-3u6U&oG%t&+VEBYBBoN|FuDUh)#thmpPAJIVpCHmSAwBSA6Q zH(y+u5Yx`^^A-J@g!WO$pMt0LuD;N3*>|3E1!oXH4fSmn`vH+ld>enLw{M4;EuMh? zlhpLNTCimuDi9i~*rY_35MxH7&feYCzkk5WbX51Q0*))se7&OoHoBOX<7ZYCibqGhKwMrHbwwbSHZdFF1S%tRHs`w*NqL1;Upqs8kOG4H9S}x z6-E%}&ZfNcI3*;H>bc?QGYtljtxEVfl-YpSstwEWCu+so+!iGMC+}H~U?9716Nzdp!}$ zXACZLencH(V*%sc+yrGL;^+%CSQqU-NVcE+aVoKzw$-){^~@1A=C zySr@}Ih7pb_ER`i074{Z*XjsSR)YlF2%q8)g|EHKM@t(E)98*|>bblQvOjY&@nKt2Zs#^0uluQYhreT` zf1}oJYh*c~>-EGQd%bFskUAd-d5q}2K)?U0osC@T%j=l(^Gj?E(2M2Yyg+A$+B&_~ z2L8r?uo!X%xwIi1O8DctyXA~-eLVF-b0~o|t!95x7C=l7+NKLBObyq&L$vJdl(;gy z=K)B0tj6N8AOSbrFI0O0Fcq3`mHVLT=9w$^3k^fO!LK)wQwk!!gypxjE`iLU zB0}7RBm6wt$_8C@lgP|2e+fP`&T8R1f9ZXD;A<%Hp!-aQyNi+v6BaWBa=1v2ElbqE z3ABl$V4o6OAQHIgz=zCqRX9DsK1E08Vfuk=@wK@W8>COmR*gHuPW!yhVO;VkTL|QO zhE!;@iQJnwYqb9(Spn)(9p;c@j~}P6wO2QnBnomj`JwdA?2`^r?@u-%C(5Pz{4=AbXXyE(h;+p=)s{4vS?xBm zAC+oA&8hJbH)Pa~4~Gw)BwPr7YtUkQe|>xt*N$S@5ugmU;G9q2SLNtH$Z;-xBML*? z@=ld`VU;U|va!F`n-q@!i71t}4_W(^0vF|c@-rRf&1my9cb+JjTk`NjBm2VKahG zOHxvpiJW*zeiwoM31LhFDUUsU=53AXo&+!XmAFKgm?^b* ztt2m#QP_WFHYiw>)ek1?HccHy7U=MVpdcoMoZ)0N*Hf6G=_)AmAw>chHXy8`Gt89EiDV zLH(9x+rWLfik-_(+|CJnl>G&U#gs8wZ{kc-=8h+UOO$b3j1#Yf4J66%-iEWM43RBi z0^D`g{{wwz{|EZSE~fzI5H6Eo@||G=arNGqseR&Mckq7Hj3N-yZ5KRnZ?7hd#@+FI za|mjGLah2nzM0D$Dv+Kot?;8{bkqe1Gzg~qb4T>4bN&4V-9LVm+u*uOw1jjVmj7AMQFrvC3nA568-zf=C#7R1)s_?~KXDir5|Ia-G3De1&tL!+- zQMpGiy(kDlO;XGmg8viN`hTFoqpwE6qr`Lq88ZbNiDVr+%UUt^O2~w~OsIVkikYx> zliMa{wzc4;p8UG6^ZkC+w+aT1QevlaNZM@YEM$`;s4X~`Vr-sDbF*Lx*l$Wuh0QS?VL}n%+Y5C%BJf<)A zZKMX-3a$~ii}d67CG=UW1YwKT&lZW0?#0%emWSeUTE#s7m2v?Sg8!-@eL_vcH64L+ zU7QPz0nKE|zCJ7$BbO(7{*1~DIzH{6mUYfnbBfjsCo~xBHBiNKeSx=_ zOKrKTaP8}vu^S$G+XKZH{)UfMH20oc6VCmY)ZY@JBW+gKVC22ysBCT^#hE>%So*Gv zMQj)jo$+}S{WDk=Od<+oupNf^3ilpfIO}K@mRklnDrCRm+ zhb@zSpQ0}cU>#J|*xPM}t81)}GidD91lo9Z6p_p1s4w{yuYV5!5}Sn-7m&p-j|oLl{#|&ry%|v4#*Xs>*Y&%4?}S`lVk*jZt$0D9 z6WKD~V{UOgLX?)YA%SyXMS^@JN^&N<;>F=HzZ=~J_4?DC<+Im0O#-YRAA8)jo($#y zWKpg6Q8gLLP@>FcK(MYqmDLYoM9P@NO6ohM(^F@!)i#?LlK^<*}<9ZLxXv z8N!;MsD`e7Vl;BuVu0!lj1ThHI1)AAnVK@wUaq02htK19f1~$_uV$V(puMORMqiyu ziDZiBZnwD9f@pJ(77aE+1UDS5-}mfaj!`vdE+v#f&dDb1+a-8B&QDQeBvYhgMn>nb zde|wP8e0)e7Fg2JJp}hCHCvr8ZER48muoAL>#svJw}ekUDuuk7+@G8s>V1f3Ws7xN z5AQbiSFnK}mK1e^ttF4PJZ~3jPyPd;_%Cc`ZX95&5itErnd zy%Db#b~|vhQAp(N*HkN+^7+d$01({fZK(5V*v$xD9(%*^&g}VJ;f&ADI{-w%GA6I1 z*wnD8&D#d#M<3&Rs<_@GGr?h9=z~3+>37&ngOhuOeGzLlHF%c-51;i1o%B` zEAZSmtr?;_AZZ{4DlBx(5~VJTB_peTJ}Y}q22taiiKR0-TAe^<3Fk$61Qxee<=~XF z=uakKu_+*if1uEiAj(gDx(`A&w+Y|+tiRk0i}XFZx|~7Z)Bz3jtMg!j0+^`~#y12~hzLZX;)fF|hiRRUBqXUsm*ps=c;%X4wOq^#Ac z)ayB8*^nK>>x-%?`+3N$iC(uc^5GvKu`f*#0jPmjRyBiU^}+Uf9+%a%zu zOhbOJMg&;z3c^c&#Y;}A3Z!R#s$;EUFts@cv=h&!WTQd`2?m1iQkr^ev zWLnxIx$U^c6P2VE<8g-xSMx#c{$10(F36KBA}+M?xsaTbGp0_$rNVe)1Uw#rx34-{NqM_X;V4Hz;nc<4vafx= zvxO3D$K?)v(v>(gzjhUFmMdZG#yxiKmi8e0^CqQ26xpKV(jTYd_hO&@+*G5&_<|bY zfm^Z$O{Sr_XMbBYL*pqq#9~z3E6223>=pw>ad%pg*vs*Y(@pl6S*Q*>KJBK$=)|uD z#S2OFqC}6vuQbg+k{kP2w$BAK+g``>ggu@2^wEptD9A9(YMy+6s+>>HjS3}zGVa(e zNCsbZ9?8j>lbyn)j$KD@&0B0I>6OQ->0iq&$%%&{=(^MD;na~7o5DP#kiK)K&0U;h zu#<-C9y#(x>zXiYG9*G{+=?3O*(e_$`!KeE1x@Nm8ke})DkEPYD{`;~4=KODb*UR1 zKEgJ7AYN)poC}Vp)w!bsTf&K7^yiLe+uQk*_=!d*|3t^jJj^L;L6l~b02iT zb;1Jc>(YK-3|N5%w8A@+FZCYcKclB)aIfU%WqqF|mo^TE8qO==HXW96NJuTiX7zSl zb5xBc)?gD;%jn6~?++Tg?Fs0JchJ!3w)`6|h5}O{w;lBafD-LwaU{Gri0T z(*!#oZci|Z7Z~lpPLPgEn>eaux{f=sS z`*(j%1U6t}nEr0F8gG~6nl{nTjKW2!q}_ z4X|VDPphVdLp!(9fGseGun^rVu@B@hEE+}G+1YujADf6GJ5O0h>#dbYH$ZX@o;7=^>wLn+GAs_|_Ujsp*AHA4*ql6611hHO4Ps)n6^F0t|%=|8(Nx=CMF$ zDYrnWDKCt|Qo>tI;6AMf0y%NH1c_{i9&wKf{c#YImkU&X*w3*;4A}(`-(Aaubn{0uBtYQmN)fg?@I2Q1%P9#55G45oy6UKm2Fb`MIJ zHLky-VqMV3--kNtwzIs;p}m~F9l(F6kl=5bI)bdWs(%S_*^_fSX;C zbQswhdu3)!EhmDC+4jl9n%0nfJJB*hHMA2y&_2hm-nMld7JJH6)Yq#7VjWauH+i3n z6Wb5`63O!l@Z1nx{U+TQS~9sw>hdUcsb!94hom;8X0F%Dld-^6WqlS0eZLJmHY381 z!cC)hEe6R!2#h8BtatCqGHXHe8?PKGkmmx$`9V2jalM3&XAaDz)gY(oqLh(b&RDJ$ z)ZOrJ^JQ6>&W%ZKloZjc)iHb9hStv4Hl&c){05sAZ;aFzg*kJWTcG1}Nm=cYpy+kM zXks-sJQtuyzagT+{2Zxui#e-bPH}W_QVyH~n^E1=@k@qvPl@pzTY4_Nxur@#)lxSp(C;8MQ!+aYeR?{0zJ@)FsMw}~3>lWUZgmt#kzNVqG8A3UqrW;CSwuN%4 zNSA7n*8`0=90c(;`*G6K+P$3p0-*xV$lT=SpH%4jtDgluD2odb`qY>z(mKY}&5 z7C>xy<^qnW9m5rBR;qq>tF-qzCB`$se-@G#sWrA)_ex*p`^?**Q>DP#A%6;<{A3$}#D`hNY{*V}eZ#(vFF~3N2^JP` zx{g~XWOLeQ9|XzZE1(H66+hz!)v_>^Hz&OMfvKHAI*F0|pt}5UY2$vZ4K8O2u|=Nl zy~_gaPLC_IaObyd(8>m#;$$kIJ|@w2D8$KkC-7ufGGpz^!Y#~<9JwiVAd%nWfjcC+ zPfcQCtM^=Je9wWvda&QK6qnAkSq>G>gwB=b9}Iw@LOd?cbo>Gl(&;66;@6?L*A3_sbY$PWTrr`9vJtsESfF^jFd9g4^h?nFQWC zCqvxbc0eI9<$+JL`LpVr8@8Q5yr8aZfX=X{?bbXel;N#1L3=u5&|C<(yv7Q!;+_qG zz3-$(Z49dux}JACeZYolPbDLTeGYzcf2!Wbuj*tzYRKPtHKsi@9!{29pS`_x@m$+;Qc zn6n6AUfSfPon%VhHd}+8 zr%Vk~^{HJV4Mpg!lchQOtAu5h(gG_T)KcOcD|_*>Ku}KdIj$G$k;7e=Md+G6Mz46S zkw@Ly#Z(~2GL@}HXUl3`e?HsT4Q=mFTzOAl(@*SopTCqlx6{FawTOGuzc{=`Ab+|% z@mL&hOZxuPuUE(nua+ce_V^U_&O03(DcSjNPQQA&e#Hu9J51Z@7Hq#8UO1ZmQ`dy} zrcl2lF>CG@>0e3Srj+8GNt_&ZjCJHpa?qv4)qfRy4vUskNqH0_!9rPtYdYI4_xvpbd3Uyrw$*3Op#ycNPMk0xf)v+ z6sIiG){Rn~_N#9DSj^~nSlzlfDhkGS$FCFW`~aS6DVIG5#?4cQYq?>ePMeNTiYemrcU!{2|DxXbv^*+cW%JM|;|(%V<=v)HPA;1i=KZ6mXtThYJ84(f;t+%xo) z$$HU{AEx;4IYus6T~gmV9egOvbZqq&D-Zkb`{MNK;l(?=%$H>o z4$I;+35islo_k^DtgWq3rM#w!2NyQO&c^0ODW$9RCU*+In(+{2LsynW#5^Y~hoNzP zDxN_N4_@tS%U4fI*2Sp>vR=QDE8}y7EKfWxPvYcVfZD$gYH~eyq)-GYc7O-OhMdc& z=13lmG?+m>%n6`nH+mPGp}xhYfN@@%O1lUxiHZPWfbi}s%NA?4;^O>jNJ z9ElEHIC6r(jNL59(b3q~d$Y33Sjf<6Pzst)rG9_v!*KmiTq|REe~W_(rPI zb%oP(rUkW=-z@6nX~<+cyS^zg(Bnt5f)d?x>X_deoa0VC(PREg7!IOo_L@%{)(p-d zyjs915%9pbDo8bmA~{Vx#I>!t-{QDO${k98=0Q#foAC;-C~Bp zkm^$MwDabn8H?j%fAwdd%<0J05z7l*LRP#b{OdNTMjxcbwfF%zRtzNr2mgA5u{X9yY@{j0!$!R=0YZgX)NNxLf7-D}5&0PW|!_a8?FV1)Uc2 z^kVWVD_WBppaWVZoXX{K)VTG@LZQ6ZJYNOIi9qs)rVmag)FuCdq&vKeBBPvuvx31c zD`R_MSf`EmBbJBlF$CzsLXd8zdSj*kkCSS`cE9#3@D`4gKF&Z$Wxz%|HZ7%mvYULQ zrsM$@`T@9po3QX;qn$>Xz+7lbed8_o$5B$#roYFjA|te0$yzX@E$hT=b0#7xw6RMc zY~%FGwY0mx^HCw;(0(K9^DnY@Ts(^l23jtT>8y%*_@_Jym#7-0lsttm!T@FG}Z>* z0NcP=u6adWwLN@NK67y(U`CwOieZ;`K^Stl%L9?OFo6Z!*MMpuG`C^+lhp`Ljh+ZQa$NSIe)!9n7@kR~L4~b*u`GW45JdrzOzOj}~{wWL35@mX8 zkb8Yi+VeB%#Q8;q0IsceJfE4qojJb@=B5npm)9O}RDOQKoNCr5TATas%`H$DoHmpEv<)H-Goq#5Y@&iIB= zR>Hoo@p3K=#uNDahdK(JEIxnw`cJ$}ChSv?%Z8-tP>;Z+z`b3;g(^ zWLc4t+2q~OIgmCSqa)xwei~|*`3FTV&^l6xqFud={w)c_hiTbmuSud3_9vGLSMr@- zj@5ZcnoA8Ey|r=Ww^}B?qyohJR4=DY&B;6&w$a)jjQVpO32W92JQES+ikbOw0B|uB z8jv07=$e!-#$NOL*hM@)YN;6>6m374Q36PS%hKsmSX<%vSzON_7N)Si{qQ# z#tR;=bSOm2K7<9O4VhgTIR#sk=ytjlN8AyZpdD(urYYZT%csQxF%CXoT?Y#SoeQ6GccdT2O)pCb7@|5z>I}OY zwcNRjKDn*}O@YIy%?RK1hVbEC#6bd!_f;W8iPyrvCWhP^QxzCuPXrA#-T>s}lX}$} ze9h-fBFcF4X4#X^Dns_K+HD337r2^y-+TYK1(Xx_rF299e5!Kd*0Gm7R?98D1!h~M z^Kth9S@bFMvNwc;wfx=Ss0vt43WTDz)&CPVH|`)N{LEXcBS0&qBC)~ePDiYl=5vKU zJ+ESYi~v%oW!7a|)+UGHZJ#A03}+>LY<<;EsB#}ESY3`4Kc(+BcEj2ni}EuBL`Vu- z#2ip;w@$xX%+U2Bn9NrTK`kkF&vPcDVkZPuE9A)I@s8uS^gP=#u3Y0tNDm0NJ?xUt zH)XzAZ_jM@yxvJ2y4{xsQ@4e9nh74BM~J`PJ>xPcN$PI{6`Ah&oN~OYbC*A5mzY zyh{7%)Udj!8roqT{Y`(uRrT@7LQXciAVKs7ZDhL+5GX=$9a`1Ctk~|aNsG(j--3VI zM|`MuKK9C;niy}K2&hUfM+kr$>rT0a{k$;qI|K)~BH$);Yqox%G?jeDXV(&{!1fTO z*QPF8;K}V?wYrq@4A!f$?l$Y*a(HrTl+^i@YK{ay_`t ze?bxiO5KMqJCMv*b7r_VaYwI1l^DWt5FK)?@28|)Q}945jbE;8G{44Z_3o3G2Am- z>y#sR9``G>=S)rD0g9!7&Ha$FDNok^@m|Pt8z^JW+-5X2W+T-XBs3lpXS~Q#cH%Kd zFjF72@ELu2J;<**pT}_tL7A>i(v+?~TYH~Ym-p!7Rf6mJjica4!*pV#-DWkbH2lrT zyTvCf1y&;KuT@NporVzhuU|JPOv;xRSlv^-BxkdUJij-uS7#tR`}}oBm@8%7EoN>i zZ6dm=43l18@mKOVCA7Q5_=HQj~`kQaY+xgq2Y#s}C4DmLv^EQeRWu+1}q95;w_)O@F{}}-& zCWgTgZT!|n8h#bgu7pyNj_d<_^?k;3MA!AT9E0UhcblLuqmNYdNH89cgF=SDOg&XKIded)(^8`wbgNWp`=5rcH+xf>{K$m zZK_edG`rw(H_2MuQz?p~GUJs87ZB1>EzTx-2p(ejc8fsQCqu z&jx&exOc!bY;D%hOz}-^S(e|Q;3+TvSm(V_u`m85-GKa=DhaBv$hvSDb}4!IM4Qjr zM(Y|yH;bw3~f== z-c_?!s`jo?)QnIui&(8Kgo;=p{PMki|L4j(xsr38=bY!hUpE$ZzV30?Xd*N;c-+MqXp-d zzVBR`C>3>{Q-~U^4v%o9N>HUXBbPbjr)_CoAM9$8?=m`1Rke3yw!gA`UqsQMiIyL& zDT^?Qyx?tUyUWuuD0SehcDcozOV#>K5aJVj=f60Hfs3v5hD!0%QqRM&wCA0{7pEI- zb>?3$QM#(KqO2A*MUR}KT|?e&rUeHIF={8&YJ_&rib9lLW%*W(U#&|DAh1CL(4(ca zF^6CTQ-qEUk&t89Ie1&77gQG+{x5D0rKe2d*WU@(Vd7nS+#!KtS?ac4r5Wzq{%a@kjM94KBg4(oO`kZMK{RYc5l$Bd0PyeToqFA?r>bDPwDV_h)#7O$=DkcDg z+TpL#jUpGC;?#S~<+`w~|8Q;T3S4?3Oo%(HbhVw3U!Z*lPuc0QcM`C)oWydnW{3o% zd!)Ua+*{mUuz37kAI=0_Lj zTM{mKdL=un&NZ0s`s%&rBYl-}d)ROfV70NDY02Ubpo(&OYdn@97Hv^91OkV?t4Z)1 z-J$Et2HU^3(uF#{%gI9S46q99u;=*O`Ohm09ZJ_>nIY;nQlP!}yXO99Q^a1^u<(=U zs-hJdSCJxgT)VtAL4f_IZa~`!YlO;lZ#=g*S>m%G&iXvYnfzTJny+B9-(NMHM@@dv z@`C1HP*>!JZGic?YOj%n=D#Qh$n{uZ>;|P=I4r5Ue~eTJo_XjF1gA*BOYM7}<+d=? zp9w7)f_!g&AoALABf=ka@BcKmzU2RFj=pci%{FqSZh;pgp=pH|Fp#?@q_SUG!*#5+ z|Kih3eOs5zaz!I%<$H~r$H}uKXSn+E=(T9&_X2qYk;P}m?4tfS$8W$Reutsb;*(49x&qGiWC4^aI?OVd_2 zcGpilSK6#C1HEix-I|i_hE54^C^jr(XCCnKo~489T|nRs5h{q@va}g+lT1Wz9xN;`NBo5yom96sGEEd5C`Q}xC>1*KSIzQ*2eD00L)TnZ z1W6VFo_0?xa)oVhg+^}gq?#tX!;fz5B8b2HZvgIzPd3V4b)4#)7watV$d2@*;{ymq zY*TsIVi0s>BQP#j1Za18F4CMPCKYUE@!+!I@wly_^2M=d^zykk?p%kBZh+Q9TH`^u zzJOIe|M=-=rNXQY#HvMDb5>l^B)^amrQOBkc@kLIU3DuY9uX%DFEZd)Q3&79y%9T- zp;EkS9z)_wM=dp7UUgd&s@eBdi@CAu0TicVB7fW%^^NZ23}U4HcErjj$c+^?t+2tn znQ1D)M}{~}h1S-+ZMAODQ!_#CFm8R7W*GDackJ$XudB3$*J67MSEot*L7dmXYq`MV zv5sNZnUNjEip+5@m9yb-r}Np3+KyZ%~A65qy3rnv1a3OeinSja;A30(6Wm~&2eW# z4Tx2Gth6$+?;prF=`un&Fu8RgzXG962E?Rz0K?J}VOtFLPGY`$Y@eWG44%1Jql|Hz` zAU_6ee~3C?JQ@w~|45@KY@)c_Gxjb4-n@plf3@?Wop{~x2^6<@wLAb_k>HV5-1$*U z;dmV9=(HR*vq@4w7T@CMz%w-Xl&CH_=MAM^qbF`3DYz&gKE_{; zXLd}^dV)8|W|dd*TP?6Z=sl-Q>sdCsIEA~H1Z(730 zJMiE4X?C-0R_~B?J#LU2>0;Dlc{HSEK5SrxtBBj==GjVS%PIp&e#{WXwvgEY!tS!6KcF^bUk`I$vM)IRr;~@HSj*Qi zp%xcw`4=VUIQGu_om%4}|FUhT7?ogqYFIga!|S|g=_Fs@5Ai>T)qQ%2GfIDk`IZVC zl~*?wUvrIn8p|jXxyBI@u!eruh?Q8uvJLs&!MFBf3%vudKlI;8qelHcVR2eHR1wxM zQfmjNdp#?{6F1+igZxn7sGqtC+$QUMk0+v-==-w#l&#K{4?Ow24H!3;o`gUQ_$nBX z;fW*%1p^Ci3*vKI2oE`ldjH+Tkf@2U?u?=qddvKQuJs*e>a_T9%#(urpg#|W=uTG} z-sCA(aHDrzV5Ii2=yp-wS1ZvKL7ETJQkC(4(4O9pmkWyHeX4(f{)ougO#?`F-1yMT z9YMd*-{Fb3%i4A(-+Y_WNop^h*q~_Ww2!Ak1Nm678SV$)6#ZOwm*2b=v{lYyutOq! z(ccg(!}2z}E9U}LdiFGI2$GUvWK+UN_!3TK8V*ARA@yDD$zmI_Tuu+VwVyv=xofvO%Q`J1hRx!gL{IrTe^f+CPy3C_v zSlj=CbHdxAam=)%rK(@o18Ux^s;kb9s;i1$bNHF^L%Oucme0%47)tu!UBjOs+`Rvv zzCE^ zZnGSJT8CCwebr*u^Pxw+@G2tu0(XxbM&l4eP4fNItn}0d=}TlHqif&nMVyVk>Zl(5 z-4^T<+gegC`+0kYRTuXwz6#(jF8PV|JfTv7QR4B}0ivL6Zj;8h5}9KiTZE7YO)Y!B z4R-cbev8iKG*tXQA)A-50nb9#0~*MC>(3iic_G z`=~L@*!J?p#QY_R-UG8Y#Lsc5%Bfxy@n9A*ceW(^wH;sQR93)pe$z?Apwh6ME7`Nh z`V4&dE`TsuGr@8MkYj-ZK{uugUfwe-mBQ;q6#=_+Cp4nQX;vDXGz+8BU-oB&(xZ(z zAAJfsE@D1VYVG@jvd==4_uHxJdPfr>SOa4;9G$lEG1|EOMyNe; zMB^xK<<(!v9Y&g`e9C{!TfKKyJ_+u?jR|pDJJyemK89TTRL`gO(hVC7D|ANxRE+D$ zAs=z@oO0h>J8p*a&!B$&?`F~u@~$B%$W+x8wE|id+xJzUoDw4|LKJb(u%;*6$GD_} zZYQ_chH&4mv3>CoS`^p}cJYZ8{ONd-Yuw2>c6eBY0L@+XzkL0K8usWxLp|Kcl3FY1 z?WrLzi1K%PQWnp?$y?=n@Gbf~8HntV)MAF-aWs}(kqw-gZDRd!@3azAP8}#}CZ-#7 zy6x5P)n6a#>@U8OG2u=v3I0N=-Vc`!NZ90JLRT-wHKt9y%4#BtN2nxWvKNV^gD~3Y zXT%iwzl%dnKKRBD5&RMWmQ5c@FZ8$DM>H@5Yw5zpkEBH0lWwo7>^%)n{5WWqi*?lw zi4aVNE5>g#TPj)R)c^T(9G{*QvNaiF4fhkaL2qWs3h37Ijwjo{dr-}g6h&DP#QXV0 z=*`CeHpQwVN|{_N$^Z0dz_a5oiuJ$~MnFLn>1v^=zEB^3k?7WR1<7(x?6w zn}Zw=J*OzXi#2WETq|Q6L{=U#G2}32MsP^4Gsw(_)I?T5eSgA>rtNq5Uqy+Kl9e|l zpnu0LYuFa4Z%K^!qOM+d@v$Qemu0FxmHOkWyZWhmT+Nbi)A#nkdv&nQ%%|T=UXI~1 zDU?^i-Cfic4W+AJIK)*+Jnig%)xpD58`d9p1QwRc(m_xgmJsKBO5uKoW zV~YZ=-OeO`b#LE@tPAX`cbq^|IqvKZpz-;0FBT6H5;N#Vf{fxLz6-9WUo%_&{t|LG z`#H)k09*UyFYGw^0N%UBubO*)#QkD{WUK}NqFj1P_6>V@q`F7c9V1f@lGyN}Nc&+< z>JLhfyqZ3JJNS%3tp%KxgX{(s!WUKvwiiQTTQ|&a%o{6S%#7VB=I_;GFw)c7*X7Xd ztRjKNPekq{==jTr*S|{f&BZ4NjNB11QEJP^w`IYFBEl0N^}+tc&3+k@qg2b0|4cpA z6*$JydBvq`SBGaqZXoFAvU;`|QD-Lk&fhT~8r(a3F@`oYU_FcJZsZ^0PkyGYM7arp z5jdSk&USQ7XMni1igR+1Jf!qGZ=fI(E|oqaT)A_rhx2yEN;bke!bo>!!g*^#WCttR z_JbVs@dEHO&A>Lv3=1T+(rYLOwkSAndaVk$0}j)x@3L4#Y!9)Q+HAh78JP9l=B%xy zyO(f6Q_w2&yYAlGc>^yN^e2rOt<^ZpBm(s{=t(tPK3)sYR_AJXy4>0dy*$1m4v%02 zyjkv@m>$VhYIf1^@}Sz`;99c|bZiV3W8h7F(36L2q_%i_X4Arx4pVijt9*ibH6=y zHwAIp*f>ta_>be(k}F^|-0aYP?KdxxpH#Yc^f!xS4B=y)%wHn--W)DWzWDNE=QEPK zpN=a(((c%II&)COQu*}XU}~na?C&%)TYeMZ`U4I3(h%!DgWPQ`bZKs}2l;r0sck_5n^*9QwK|>)?R6El1B2GaowmYul9Zlcu%Z|5 zJ9rl2n#0J|mww3bx^I}@fEFeJg$N)uf6)}?&^*~2$XT47P&2*3wO1BF-rX1~Wy7S$ z59Nz)?*-Pmzd#gJSYII54)U$cs#?#nbq*Jb4+$q{OVOkgMld0v!EBgIq)ej4W6()D z+u0*$w?oAL7VBSf1OK>3kliecns_IPi*|QBNrB%UeRgx7_4hjBQ(Vxm`c=xxkBxjb zxD>wwkg)4KQrVN}Jeie3tC4%uGHox;GuN6szvaUIyC|}D&c8{2jcSuDBKCR9o%0z% zcGFvV;jyg!Ye(-*7ZwifxTkF?&Tn9?3^wIRQwznmIQj5NrM4tS=m%O1ScWq|Ywg5d zmwtlz)z^7BY*oF{*+4>59v`DoUb7$g_NBHiQT5J(QP85OTypEW*#6FkeGI4kZ==#s zXTv$iW$v#E;NNgaL_Jr>hoO-qS8BB9bwwJQsJPU=mWIOob|Zp29j7~6RC?-=(;GfT zo#ezBpJ#aGu~-J5bX{(i{Xy1&!a~6SZm*_WgKOqV0~x1oIU()QML;em>L?#W-GnO$ z2}l5SvdIQ|%#;bwUZ<$mjXKyBT_^+t@z*APRmF6Ct!nqVr>c^cT|ZFT?7>xBUDuLe z=k!G|0lITZ(GU$jla?yxy>HSPb~N^z$_08jV21&xmqn=(#SRmm@nvF7fZw*c&mKmN z>j1Oo+U?z@br3YS?=#d@{VY$8$b7VROCrqqAN@EpE2Xaj5kH znxnhwA`Y=;YzyBr!Sp`}`@p-R@CKWEt42;t8GPvecEOFJ!720Fk2m>Ccz4kA779`HGb1541d?~ zf`6y`@RW%8E%|8=B{=uWXghOYgkxZxpKSDFCPueop;<)xDq?Io<}@kZL{+H_)6f{W z&@3KR(OM_7Fn_y_$0xhhuntoe}8WI}bI^_k? z?7rx!+t1^DrmU90CY8o8L^&Rs?nrT&?cI0Vy)Zrfb7*^yJ?{Yf`6bLKy4Z{`NBc(Y z!shHe>4=9mSEAG)(>e9)DN!HJ)%-&@18=USi=|#}ZK_k6Wu?R&8TNba^=ggZoz)+&UL;cD7BP0UU$jWESj^u}&hgSVg;H!Q&%x&dW!cX~mN-qY~1P8O#} zoq3i>t`E36L})Q6ug)plesJg{zKCL54au68c_QB>S-kM&=<9u5;Fmuj?_Q%x_k@^n zMsQ$P{~K@6wH)3AE?o^Jx{$$axrE`zOem^XcdO1oSv2x>j_G+OI#PRITfKJ`<(IS0 z731JxGb-?3gMyIzh&6tm+`(qLduni5OhlA4|6~r={v_Rxr;LZpQF<@vxOaqRo)2IB z519?oi`8MQJ5(VwUZ)1|^QUSRc`jEpAs!ru-HWh3P4kf_EBG--a4@nQt>!Pmb3nPv zbtQvxkWlUbvb8cfnKdrjC2ar2+!rsw`!tg=k7oMxLzM2Ma%z!^mr#j)53fA~sfe3r z*`_`8PE+iiGMAO?)g4JnDsC>=ur_ynn&006TUK1^3U~&`#7#3NX4K66WGm%4p~o40 zS3+%R1lIEEh2fx|9`K1DN-?Q2kmJ?tx&T{LNlnB&SYE7{?caWPcw_ps07G|fKXl^*`RycPfAd^ARcIh>{=H_CC;958)^r(J8GgYU5hAm0Tdgg1K2wqo}#^i#6 zTyUgmBb3BXNDi`&jk6~Sw6&L+B&iklUZnhTb~r195~1hnKZdGR7QO(tz5q9KWwI{l z^Jcj^z)oY)QuAf3mwtXG>dQSK!y8Ugk z?A#;(G|1qkkyBu?ZGKL~k3}zw(+!~AQbT*n5nFmqbx4Q14LM}Cfm9Y0R$z0SZsf)z z(Hs_PAGntvQjz1o?)ELIj}4g-c>s8R%Ys(2PE|u3DUgFdNSf-&i`s$oaz38>(y#HA z$}VQ9bZghmN@zOSH>}7hO6hNEMCYI!;iuV}&x0T7oI=Asbrgii@svo9NYZzYRtT~W z={IefaTdC;s;ay5aK~@j#(y}%dcdjJ_nf2d;?MPjZ?u}!n3tn}2W!yo)7Asdn|=Ip z*Pd_)ZZ5EZ^Gb1j~j%7dbkC99>8f z+nH%1OX;Id8@VsRrMy1H^Uk+62f`G`aEe^RLiU@dzQ&6nfvQb5B;=Kyo}vq z9l23oOMr51*yQGIMyTjn6TdJO8)}j36DKvgB)|1g7KM$KOxK%a=gTdR)GvSEpkXR> zzw%g#PL10f&zE;Hrn5+@zL=4GhHT%WB5tw5kE^fQA@gMn0a?B&yxy@6t>3b_Ya3%X zh;*Fj^0-~=t#{^5u&c1ff=rl(*9a)3(r7iYKxA*Hrrco7OD%8e7Q8LOx41n1%8^#! z36OusJqf7o_i4YBe5yXX(6B-a=U6kiJW}hQl{;;WSh)H$&v{?NVg{LXzzi3Gd-qz0 z?e9xwbG~FB)8~5gLJx>d4x=RFjP)TqC%~&ioz^sC!m6H5y5qrv@cjBnn)u`FQ z=Gk$cL{n-##oR$7wWLR|l==~Wr zrBCZ|B}9#sio~GmDd>)STNS|ZD9VRBKQhd3C|IA8@QLZ*;w0VReHNzwng1(O;1#Z$ zyK5ymgJw23o!qZr`R@8Sx&qex-(tX>n#egg|Judb-kw8J;wLVjWrry1MSSwf(KqW~ zG4Fh%8#;uU4jyM`%KTCI>a(JwGWVwb8(Hq`G1b(0_O_$I$VRdovN)#kqA4ur zE7PqbKmWb$MOz9sd#et1TJP9HK;E|tD|LTy_$+c;&hg+gmE$&c!I6yKOOpzDjc8f_ z#Vd)F3VD7vWzoyl95AfXIGk-6b9cBb!e<0Ds7#MSZX9+9mMp7}UymxuiU#kCI|Th~ zUynZPt(UYhlH-(zB^C=VB|V6lusQp?j25YTkt^IIyxb~Tc1@Y?YACX94wS6nc?@e0 z@>p3rWq#2eN^YFa%)caVtnr4PB?4iWzogupx!-n%OuRdj?ouCcjIRG!TFX_SN=RHb3`k)8%l zX-jd10KYvkVhe~*5}p>Xi=NJBNfOFTTj^1rbv-Ku3 z{h)YcfpQ6;?s#QtK#TVzo2$0xpz&fu?D^JTApEn=~#@zXuY zh#I=g=GL2m@0sB%hKht^6LhjaMyuuu-VP@{-bvPpZcq6Ss{`U(r-hvg zKwfQvmut$~;CP(F#^Fu!I~72F0{RPmnb$No3$?vT*b`Hw=7L~82}A@JS9Gv@2Vs*n zP?hbe0v~k0G5xwq$J;&t9X%@do=YrtnnkA8S5zG*4KYL~xp8o$?CBz`&WKa}wh^IX}ssUoyVfYNiTPc_J7tGhdx zGzt4C^wA;pP)1{7CKTcRyoFHbpV@aHdGBCDysY>(e>Ft+`FBA)e|EHs?(YHGv+h$? zq3?o6D=z776Z~ISwnUPg*-~L5dBCH#7=_a|8m|~IpQG(>`@v-mUHIr_f4a@)nhV+*Ecm|JcS6uaDd_{Ce&aD@A=w1&@XO<%(_IBG}yL_LW~xMkyy z1-Cx$jo9cM4h;T%i3I3v4sFm$Y{60W*y_sW_mC35bn2NO$)?JX>LqwYf%QGUVK^3N~JHE(69YyDfWU!Hr`q5kZA+Xj8K5Fe8Q9_qeai&1}_ zNt}jPY}oMfupG^@!Uv+K8Wkm*x&-Qh%S79P>Ly>Q`A;YTA2;@t=7mDRyE|C&n()r@ z1hg~OFxGRO@i5Xw&;?#QS2Wxo-f&LWnj#Vw1cJ}sg8K<_|9uR+FLkEzLAg$>ilKcY zVO4N@`JHpek5pNKI8Fm1K`AyQSrGPfq>0jP^ZRJ3mEW9ed0Qvz30-l0#*SxfyME`0 zX*>h}{_dGNFaGx}c=a#yVwMqkS}j9vv*u5F0Pk;jin-5*tZRM&FboOg7ei%-M&DcG zJQK?C#LCt&2ghdPwA;2)cOv9t;~kEl@^~N_CYB_GYATn@>|i+As5@I#G!ld7Zb3kf^3MR-mLMl8HfjeP79o|%kFR^QmGBlzb?F<-WDbLyt%80&Tq`}Ulo;AO2|eB= z9r>lVC#qP>n-z9A{>q;^Q>&il&G2vwdzSxvz#X5uHe|_|Q5)Q0;u(U$Qv{w4S%%+vK>eq%_^4+u5Kt}AqhjLdF8c+wLqMQ)-|9V7HbH*{XS`_d0gutClpl7W)%(jx4&7 zu@i~Yh4B-mPx7mT=~IJqLLhgMC?DDICZmJ7hjnLv3H($f4_0d{%a-KSjfuuG5Mr3p zj7KMm2Vl{xWBKZ{^qXpQpbW)3cQ)+q_X6luR{s0${eU?K<)28e2iER?LCh@%3{LB( zgE%IBz7AWoWWb)AWOUVoUc2XlaK3W48p$(<+vA* z?fb5phvLRGvq)Y`c9}?}s~yj5&0^mYlNq0QKyQ9ZY>Wji5W&*AmTziYwru7ECvY!v zU(DZzez)(XN!svLN7cJ*wFDhW){&xZ*CO7keo>v#$4J~+ea@%N8R!JXUfjy2n0!6j zOui))V_+Ly0^d|28GRmhAxv$f6m`l>gEXB84)fl5oXZEG*bsT$1pb%@@MN;AFT0n) zE|b@KE|hy=%5Ig*3myM;En3a8+;dZG;Td0I6(1Cj`t4lHuRZx=-z(FS8zF>{z$3k0 zQS_Ct#`nU495e2Y~B&s_<^hqvLuN^V+Po1Ry+L)5#H&0xRi zAkkxEnvjGFx9y##xz!1gxfAq6KMXAi%Souc_Xt3KwM1|XIP|m|xcZUsjXkiGchw`W@uCDU_yU1O;{AryW+l=APhW7nNUdjB9z+j8ES7x)yM-H3HFtZx3+3s+`W2L)NaNWC0pXFJL+ zig6AICB-c1hyQ0Aa1m*J=eQd+xct*D(5P5zrtLdx>#7BOmz9G10bFZUF4A3IJC1Ol zuE!|1O^C=^y;CKO*3g_`LJ9SRwxx3|fASipd`0!0?PVc+nD%5yA-qk8K3mK_`R7FU zA6CX2A_Fg+k9Un68VE0FwwXR{sk4i9#-)!e@=A{-?iW`3JElm=78>vd=7CJVrH9jR zSpe|!O8eN7*le{jl%4$imGNI z$Pr?RAq3U55R*q{Yu8}y^#Ej2co!^%Nrmr_HA#c_IPP<~zW#;G`L4#ILySm#fNxs$ z_wkP_pM$QrM$iq(^d4E@G#AQjj|Zjv#D{qTalAIVstn&UHxuGuIG6)1*R5gY57 zM|WVQYzN2!ff7JB&6e`?GlNnk@iQ6WjG;re%KI9DCQ-}xGRr}H9iZ-2WE@&EqmI@f z#MN1Ibwoat*}729#VbMalW}BV)#CYx+`r-HOZ2tzJuKY6dY#`le@Y2yFGt64d<=f@ zNBNEPoNR+0@Qrj`d4~6JlX8qRzgBP)VdZOs-MgS0N-o)O_h$V1{RICaii~V*YfW9P zx=cC^z$~>UQ~I=HE2))6R``b&nAPsMa|367XT?w{;Wb>iPyc)~E{&4ps6Wn`&CFT9 zU^i`pE--3@oqg097w!;y@4_#jWT5a|rNKDUH#I6y02~z^0C6f2cev^e6!-;Whw`#1 zcxZ)XQzdUaOIfMhQ31|A^|?)+nj;-8k*5$W(?_lW!xTb)M?Ph**#0uXfy$5r{} z&$e(@R`#(~da~ZP3$Lu+Jm6`mt*G4Tit3__;T;W?M?~c?wMP(|3~xc9S)gt&i`M>B zEPV2!GOx=?@uoP`dotA;R*al7Q3-U)gS9?dXuxPJ%%;gK=6-!`0dltTIIZ zKFVrng!>m`Wk@*?nn^l>=aqrohq;q5fbcl-9R#H`u2&`d`Ix&B&n7EQKx2 zTLS+G;p)nL56!%$r}AWL+aZ=9k78~>F_Cp&YbM>u(lTUX1i(Blm6ku!|BY1?JGgy^ zlmq0@Z@=_n zv*ivJ3xDeatGx@r@kkd4ATLk3srgUXSoe~fnkvFm<;7gzJov3k>KE#pFwu3%Zuq*C1)uzksp0!Ob!Je#FA(Ik;^cgFyG7v%< zZ=JL5{1eF!*xLL3=^BhrZYS10oE@F(?N$H$A2(g80enXB*`ngdsFNZ@ZLwZz5;`;#zgrBmFD$%_|L@qnXTjk{|wh@R8h`3 z6=F1QKgU+SZc8PMI_AUJei zmUG{jCC89PQUX(;vH$&$EZEAgX*-d0pIx)%zJ;Xjb51m`L z$)*s&Ygyn4-Z>NU*4(OoZWXUtU9qP!fj+X0f#ap^$#&wy6gK3Y?l%u2ejp$URnASXd*)RPwuOL zoiAVw7CeBP!(bcT$EP8|?~T|7Kspb0;MqRECi-t7R-Hp9{AX_N&jibJCEW~4O4fnf zTZOLslKK?*5_4H8CWu-4zpL3jQ~4%$2r(Yl*5sh~n@>mP^~^ed@wC6_sfC7uQvGs16LJ3CITs|InFZ;K$NzAFwAt~f84wEV2md~~1T3w77 z!Xk&77>t2iYl&%4#Ce|g+M1VCEL5eXa)PEnf9Nxf$IAXrMo7E+J0l#-p=bEt)eeh?ldV)&u19Mgv zs(>B&&NJs&UQ${q8f4!?BmKuzO=TqjJU&%bAePEz!Ki2YjB2EH>Zym2E$hv~(SF(; zQ@+?`QQnl(5-Ei1I1M%Gk?8}@CT3UTy)r?*p>(Zc#-~sTX1nqYG)@c^7*@sPs>m+v zeq-99EF=$h5155R!X}eg){KRQlT{&{h(oQ^FmCrlj+`?vkP?xXh``WjnOw zviUGp+HJd?F6+(c2)7gh^*9jMzZ*jmu+0BUPY+y}+%-*==t|PqGX##_!Hcq(S4#zBB zlR1Km7i+IQWBYMZr>h^ANds=vTTi}q1VU;>1@nO~gO%AIJtF0QZxJ!DQ2*2#XJ~f#I3Zv>S;ESEb>eQ5(7wQsoFjm7`D7s> zX7w(R2AOz*11LkeeXMcPfBDof2bRDjL1<@DW&`2h{8as9tQ-1#o` z6GRv~=)DpcEZn(S-cE!aDhFM6xDh&}78qTgHz|e^BPeTOOH%uvTVc))Q$fl3zC>Dz zWr1=PNI-q;$V0|mvG52h4Z}&dzW<(gm#0CN51~Yw=$Qfywf*aN>o4q6?#PkNrfV!A zG#*#&a8NSxR!#n^%TFY2bppI|T2VfAB{CbM$%e31u5JjXA)aJrLuZSRsh`Qps41r= zoN$eqSPP!*2fDxfx7{mZ%ejzUv?&R3Yx@0<*@JyvXk$ zXhX<}fx%3=Ab#4*z!+IzNSSNYf(K8Q8e%39psHKFCc@IkX>vn%**h9TcC^aTST<|_{)RNRt=Avy?@UaM5(fKvEP(aT2 z1l};U##i34g9o#ey4)@`X!QwH?4(C7NWf25W;J_D56M{v2x^pp$KnjDsEKlujv zHixcKDQ~I`z0JD0%%QzH6vfIJ{)*4}zrL)EdCRPa(Sn`7suNf3-^yyzzhj%P$EXI^ zwTrstNWElMv|LQo=-vt4;IsxUjNVZbD_P8clEHkWZjschnsq}K>%WuqKz7nuwMvs$ zw;8knll+_{>4 zHPJ8KL7d6ksPi&UiC3q)9~WLqBWt{J4BATIHFNf@qFxTg3tMIM*fd~H zOiev2i7$8M59B!$q6WB3QB`DGMN(IC?@#K_G3qa&ufnWK4BAmX`NY*2!H)8!vS2ei z_gBlpy!$GN6kQxscVRUJ2Z_Zot623NiwDvq=A;~3~v>6Fg{px0%EzorMDnT^{V+IYr< z?uW`{igLA70V;n@)XFVyL>Llan1MUr=#K8Grs$x*7~Lyg%wf-}+Toe43Ox`#&qeO> z{TobSU;4bb>_A_-a4LAJMoQ z?b99{|4eZ5P{~=}qWw=HAwwawIAlGQ3q3>FORMBM%BxWBHwkN|_LY14UF~qW42AO0{jAU)w?!c;c-kRA@tmLnMB3t}0qUkQ0 zrW%OPnAaR%<3P~JYQ|SUcTrH3AD$*8RemdJWfka)5gGju1I@n}SQef*QQ>d?0ioM^ zPs$cw`&}bpcR33UgLCM6QE=h(jP`vr(*H?3p3RJJ zQ)Xd4<@$Gn`HeQc&^74JN(+J3q~&Eu#9CIs_=+jnYS`QZ$o5m!?o>SR`4kG zdpw_d&F8Etlog)vU@-9s1Qa#=TC{L}>G`9@ojMuK7%4_YbU$cFN*4=S&1p~4`C#z9 z^~~^YMzULThCNDO+s0m5u@qgT3Q|@Vrt$IfExWrMCkHgTlQ6@dM5up}ePO>ZFuVNy+J$D>w=_p-npi?2H@2opV0X4s_A-z83fJU&= zt295ysrxVvUl7HIBLH$C73f>_MME%djzzikUSg=r{8z4(=qfFHI~q=2>Hnc`Fdx1; zA8IhQ;l{Eq_)B1S=BcyZPHA9O9Sv-!nj7u1t8|13a7_g#kS>Xx6PM7D2dne`{AkYm z^^3rFB)6FF-keFnSL}Qy`;8h&?(7oQ_a?$}!}?!%$Z~0%?$Uaq+a*I&sfZc=0bk%~n~}qAg}z z;Al+LAQX8*0TzT*^ft>aUS|Bqw{u>yiM5e~epWckRvo zRoWHM9*pY8QSv(8qgi)U4qXzS*%5_gNv z>RWIy9{M~b;p^pBcZ;*4-Bv%c4;%H+QT!RYzjiM@-mTL{)n{m&uwK4sl93Z+mu@+D zBzh4Q@}v(GsM2UtR1nhk??Xe=b<|StUC-k-8@x97B7xbz_`5#XS86F~gWb_+9%^;A zpM2q|!Dq)7v$VMJI26Fux}uw%%GOxGst$sd> zeOdG3slimZo)Z`4q&}-2L`5Sr{MOx%&Tiw6`P~Q7oKU64(<7Es;caHPwtNKpp!_K3 zjsAAOE_@fi5Zt#u)ia85kPmt%!5cn~@{KSVY zmlx3@l>x*it)4FDEALbjwq%uO%hE)gQ>^^%8_^40<&$cQhgWW`sN6H7%w_e`sw0u~J(j%o|OGYu>v}&B+zCMRKj`&z2zT zr~XZf65{B`?}AC#lq&QMtVk>e@rgY7o^G{-wuOtaXP|ks-X%ti2B19dg`WwoHsTzV z0T8_qd3mA7|2xpjfc{sGyxly@Wnn?U|B^I3s+c|&B`C7@r!^fZsO_fPML%c_LUg~6 z-B0Z_6r7XsZA-;XaYTsIjJhdxFY;JFcmh_^lRW*?+>vu}&4yu2*dYL!wlG**KWE;ZsqYO5L&${fRx%tuY?S1kak)t!Ib0+_@N8EQXORk6Cz>jS&@>}RGUi< zqly&0NOrjr7t2w;AA;QuWA3?;^M`_F_hpvy`1GJf`21L7CJOgW#}bL?WkR*5FKKyS zh0QHhwyRF6WYzYCB8_fI@+jsVC!hD~pUYdvnj!)C1NqR{vSlrF_CNW{FJ}OKDwDx{ z3JljNyD4#N{CLjiKDU0yAkvl>U$5*H);y~FqQM&*KyIhP^gb*-v9Ijxvt3U7lu6~I zbk^m+%HE)Ddf38ZNH8EgJyFIX0P&69<-5->%&Q20U>)M!ws1sS zr+iI&;YncI;TWs%UUozBp7{^&=pt^q`%2?^ClaqYtXp$Qy9zBUF0^NUdo5Mxt9zm2 zlpphA^`GGWzoGfd^EZO*T8?kLh0myN$=+ju{~t%^;?IQl|M5yCk?SWVzp%%4_CDwRdOe?| zwu%i;6i*YXGx{%$gcYb>+W9X$rh!HzEGzH8j=hP@1cfXxM(}q1au|X((|L>1i4&*?Ks>n@;)^VQP%;nUHG7 zAC)2!Jizaj#96V6K5P^D@xZw$aT*6FZxb&i z4vrM-iu?bPDPk?iFM3QAt~g8XzM_4e(wnAU@f@hopNkihrWOroO7t&$J1!d^#xObs zya4ttb0?}_sh=A4%e;9=AO63jf$E{pYf162jG;d)$PJ}1KEM-h<~8}7{e!O+-gHQS znm@hIyt~)okObNS*^revmfTSxiR)y0?|(tfo9^rdF^fu^YFpPgn8>zUFE3EKxzp?; zSLtoWw+G3p^9@a@Mb+NYY!EwD|CN}Ls~(ocoWv)b9p~ll(9-BZtu&Fk3GcAhsenSy zj}7zQ2}egm?^2ImUZ=cJk)r){zo|)!$*|mWEZyLRHif_=P9`c|-6!M6Q`sJvoFaMJ z;bREblSMW8bi$sxJj2h}q1>qw&C`}x%#T<}6{s+&oN;swGXvJC%pJC)`U$w0EPB5I zUofdJLk|!1m4CwKFw{1_)0@N9TR5gf+voe%gjm+Lf|G@Xx0kbmHtmus0&kJlqB1L< z1RZ#|nEPet6N4w5-WpGbUlttFVpa)eT%&X9*RVQ~!|of1xy(PMm9`#^j`Y;27{0$h zE0rPba<2*Iu(UbfiB4sP?92EznNicQy>a`N@X-5f_L%oCVKBq92S7f|6KU&{Vf^bL z)gf^`fH-sebcbw?1=o=T{^?{PM^NBgQc{&h{+1k)@{!}g{KFGeTtnD;_8ndTaj971|An`R9Ay)~O zGb$F9r;6ikKAmjv#!fr%?D*-xsoR;i!bMfh@V3cl~22r@HN9KHKX} z!4FZ8mF@iDPWwgHqn%ge(JxsydIr2k{0SOp4n|}aT`g;R^9xBs-p5a}Pe$*m|KOsg zM)HTaBz$(!ex|p@8E??rqEfXIUm;5dreR^IkV}NYf4yT`&zM<8EP&pH=Ye5g69sZs zdA==kxX8B9!tgrwp!}FY=ELgjPBDa^=-~EpG9F zN=vqGu6z#h&8xrmZJxEYdMnpC=Q%$_YD%<|r_=S_+s_6J!b(YLNtcAhed4?7!m`{B zUl!Y=x+V_z~{@f-+Qh`ZC&HDq^<>;!P#O z74TaIO=1ohCTW!WzqU>Mk*~iyJtLyS@p9TKmJOdLNxbuPVnm<=c)nZn33xwD+0I9Y zbOgJ>K_foike&4)O!-KKIGb}|DF{<>QH)f7KN?Q!pMsfF%c$E78?;Q9aVL(Yrh1E0 zn{j;yut2es&WlmwqSA=-R5>feI%Z4|{^|a_tv3Yu!N*TaV#CGq&BfoC>C^zifcpvT1%F3E23cI2wRJ>$_l+$eom`rO3O z<$Rg#=r$@fwiNb)?v(!alCH|D-NnOK7Kp8CLN(K(iA2ya;L);DrB+2j?P+Pl7kHIG zF7OqiIJ5eAoDhdtuyVWTfPxR2xffGcST16ncUtyJNwm}p>*-+1t$@Ml*nois1LR;s zIdY%@9}ky=IYFDq1~c$({T;vWhw}FDV$%?%iZegj&6_WDOTUEfi%$A^aq&dxp+%W} z1zQA;l~=f>v-NcRc2E0i4W~8CcdcnB24qfTXFBa)Q9v=##S=u$2ceZ~s7mlQ@Z4<2vP4MYxhObgJ(GkH4hW@sLqSBDna=&eub|%=yivew96?kyfQz{e0J^}{l)9`i1`kd3G+ub5lUo!)OpMmoGpjWedR zmHP`VdBbrUwm4?8kC*RdVa*ZvZ}pE37X6F#LtOrWwmzDgr`UXEdIO+7T*98iNOjm; zMhDmPKVrCC6{;WC&mVpDM$4ibpv52lTSAf`BJseTUDfJ<5lL2m3D4NKYuR|G4b-~B zLCW}}aivT13CDKJdTC7D`-joeU!4;L!chai&eaY3;;w~cy1Ymge!ToPkzZ?O40TAu zW)InzYBISpP*m~mus#^>woJCW@pzR>fNKBhnKN|EE@d+V`_#XS*(svn-)Gc`4`G#$ zxuSdpm5OdYUtu@cP^Bfp+34(NV_(P5+9LESq10&W(LF6f-FMeuLBZ!)7I&+ERq}9F zuIQ<+&&G8NXo^m>HZbl_t_|d0SZ>Phk575AiCHItqaVzDY}S=J(#|qJezD$T-S_kS z_epgz_)Gzwzi^mqiR@bit}Zq98~Yx0tt7TRWLoWuqdC6SMzg&2puGXk6f$DLg7Tu5 zH1yk;UjZlTom%p$&22u|o4joutGYZMQu7R^A5m;ETKzzzQfzwDeJ5DfeOrnIv8MKv zxqdEsR<^!ZM+aeE6drxA=`fG4LPnI3gR*Hi@AQUNA&7UZi`3`c#tn&Q8hktb8C*Eu z95!XXA?tsjw5QmMM>GOMdrrYVApWVsmun zMB`6FTdk($Ql4!3tv-1woY8q~YeH=f&CsSF__*zAPRQ_6VYKxsJoYbWGp9pF(jp8; z`%Qom6J^vFFCEl~xa+kCfm^McEUWvRU5YDRb@J*f`e#brr<2TbK0pM?$epV@ec`ENbv==Qj^vDNeu z=x*~6YZhMREI?+c=iBhg?v1Z;QrFsXJB-_fT@fCE>ZjJ4P9aUpHGuW%* zvPaSa_Etn}0&dM6W9Lh90^6Z#wD!<%RRY_O^H4v&z|&@=hcXaV;aP70`UU_@?&vdk z@uYXr5JyhSy7hiDA$z{f+zsg-Fl!lwH-ddlHVciYLXso`vYz)=t{#KCB_F|nwq?&7 z(myKX@WqerAF+u*%dC0?_>HPz2|Atf>gKFg6(C**-B3%3hPs#kZ`)Z#or5c{e zTY3NO>g`uYa|q)ayuK>k&~8aI=8>iQ15UfEKo+Q;^c-de7Puhv+U#Pv(&FrlLGYC0#3^M0$J?mj?_a(G^~Gi(DXEx1FUs+xS=B zuP`Xu{p?|m@6pRU!Q`)$e)Abwa#r1q{dvs!k%Se`=$#nd)l0#{=#-+1$XF{9xxfJr zD(|PSer?9by%G$(G-tt4e@(5(Wp2>-QNhF-JEOb>7uVk8823u zfjGsF3O%G>g$PsoDuqFtqGV)#Q8msLpfQsRhs(Pzz|O*=?HFZ)93{b{=R2L$%)jTD zg@OaX$ne_98qg~R$bNUm#mEw+EAyJ7PKIQ4Zrl)@&kq*`?j#IPsIiHSU9Af(96Sa0A5MQGWdQn$Yr&=rqw1%eY8*I z0!MVN7+o@Mrv`q7F|Tu^cf$hzrMfv}DV+a2jVmJ=uTB`*4V8Lq)cGaO&_7~Hm*;z2 z#7Z*xu#`m4PVcM?BK-2)jKF~0{5RUD<+t4$kC|04DjsGIBOibuJSl+8DxNsLj*+)p zWjeN8@RGMxUN*NFd-NRkI2-P&_0^`2GG>SI3CUpsz1EK#6zzGN5ceW>?)F_@bKz#$ zf9Jw-#KnX9kVF29<_+kd853|yFhPExTYoOVWWjN@rzfHdD`c<|J-GaiK|zpubGQhO zJD`sE5!{=0%=K*f;xS%v8uPC3)W=l=+vY%lHLPl+Qhro&Pu}xvdjZ0hCFSR^6{WCU zcHiUMw7-z3{XBmN-CzaZyKH$K`qk9Mz-I+4Ttv5J0g>xFRbCJ5YG(e+R8=Rd~7|0p|RWtu-tEL2%PBlDUDa2FDagEUBe-)gSV9k)Y@avQo;E7S)&1(>DDB#b zL0DP4&DAxO$v5VU1-ZFf5D%AXAbV`!-uUw}ft_{=&Rv)e;CVl+j6??r`#kUAIZ61( z=q1lAnv{3U^^rT9fz2>xKJjiRwJ*nY)0UmNCS#5shMxI6=B*A~qRdKELy-2l`*k8sN}U zid&=mej4!tJ^yYd$wi2OL=bvWRWi>^EvIzjq%PM)`8{6KiP5!&rfG;>867*_T^WZf z(X;6wr=EwuX!D+m+tQ}2%R!m1?R}K#;=8_uc^9m7$mdkHBi+N$vcB>VAF+TmCirleDt3;}3UEG9I)l=81RnVYFuxzyQjj0fsLg z*vIr^1brH=8aHZ588K`p^r1pjO&e(F`5?6l#&ch4^KK<~^Rr3oFxd@!kWrhKC>vEx zbKrWymIBKpmG?`J6&Y>Ikl{7hy^cdmu)B|m2$nx*&QL|={|^9_L^Cc%oX0(FOn$s3 zyH9%5e=!1P^dcUfRt6*F#s(Tn23Obo<_)&iwj^!{4pHpDqN`%p$xLfUJ=l*j&8s49 ze`dE4o=S+rbVQB1ct35r_vAvJWFH1W{-nt^)c*ZaRm7wtW2&1kB_T;J;3CYdIf+0` z?&ap!=;W7Gi;yobibZbjWhyJ`=EGKbgRL1U>I!|;e+Ar$g_3tU$uTt!f$F;o0IGkU zz{~5pKMOK;zFAt2Q?vkIo-&gV+FH(nW((@1CZ zJ-Pm(44%Hea-Gh7f|kl3lJ5OJKk;7YXWNc{>4nSa8nocC&q5$~ALACA=C*jY5GVjPT)r`PULZcRpzrP}v#$J*m#0=<8Sv&c8n zw`S5<+$w5j1-Y0Z(i?pULe5gYF$vDZvaVRU19C2>n;3|97N0jP;MPm*c zB$KJ#f{)ZLf|1ZRt*C`II;SFx(xiL5-C$sM`sgTy+CF$iI5$7(G7uf$1O-_uoW{(& zR?VE)`iN34?~ssnz)gCRSM&#-x+6;#$X{IH4EQn)%3GfYzskx{rr*xhW~(T(HJfck zgFUAo+DNZwNxNd4dzFOtGmVq8EX#rIisp(T3wyZ^jaMQadfrAQSA9K6(2!!IAM9?C z58AKZ+TYdjRqtRzx*K=C{_x1yeD{>uUD|qzzNb@k3+g-6s^01w@;k1%X1KEb&X6d5 zWpfP`P_K+T+JohnDb|%PP!`%edk+Ez2i8fcKBvAZBIi3_c6H9&Va9-s`0X%q(uZQG zIP6Nh%Wja(e(hYf&+c7r?i7>nl|Qx%$|t0j{Ch;G4zb@UBld+8NlBNPn^v$e;YfN9 zVR=&Db8kmJF~t+k2jNJ4KTc?6iVd6lqsCsq#+Z4N zt$elk1Rz3Uzjv2`NHc)5=0X<4DS%Zk_5F1joF6;9%jEQY4%@Fj|Ja40z>iA{+li#I ze81fx3rev0>LGv5siUrKbB#+JBbCQhmp#1Skxu;n&=+U5zD%#sXa%A4~5oqBiJ z!j)L$+Hv}ZTM5}q#2UMjC0r6P>`7_mq{HFcw9RNMog$@7EmvKLcTqgBAT#nV_vGcQ zpS4M<*^P6Z1!401A)ET%f+qgh{GmB>OMwaYYa2q@bQy;1uQt>hnlBOU5SErDY@f`} z@-h_RcSg8Ul?^}}w*vh0Tn`|#G5*(cvIz8f zT`=7Y{G9J07mZIjbTa#d~&R2Cd&7R*HRWLfrkF*(E_Yp;BPXv|GXvU~Ml)EghO%CyK?AUk8<_+jV&PhdycYjESMex#DGt>CFg+>>Z3_!%zR7IV?1Q zGjDSE2JaUDXG><8Aw)N>4{RfOS;d<-0KG}ANr;v@z_CI2eqJSZQrmK5AtPwk3om5d z!^N4}h_4sg8PIg_FU^OYIH>i}C==c7#OVL1SN)qL-?T4i`+R=*&#F;$F*%}jQ2tWe zv(kvhio~M(33**|YpZ4YNs4ic41K|UhE}#?V<8gK!$%36NFK`UadOldukGj-%MFv% zmG>U}FCadDn_AF?hxB&I$~T&veIP=TWKp|zt899b6?e0{kKx&5VpZRN6+?$#&#>2a zhn)se&RDaRhEB1OY&CRiT9h|uvvJFx&+@637xt2PAYmcxha~0!72n8!rUTX|S(BMD zs`8-Dkkq=|1+plh!^~5!YZp;^i^U8BU5?PJbQg5BjIf#iuh59_I*e}^vc!gwkmZSF zT%+~)l2M8cjoeinxZ{I`mUY{jRxZ%6BcYt?!S0JMk7tDL9EzzwMWpfiXv@@NRVw>OqBe*!P~u4La(km-B=8rRI}CW=@*f!7e1DB#ZN@38F& zu&dDC_Fictsej*DXm|UO2Pj~#qGmr>89Y1Zp6sl0{DtsqrmacL+{O6!$n$Ql8IKj< zIZN}K$6bEEe9=Xpvq-guDy4wzY|_x{QF|`J)>BkVj>P$jraoglEg?d9=%OP!D20Uo z^4i}E-yv|pty+2OvamlOv=Zvmw2~4JM+M@8YmQ1{Dz(VCpfl}WG^5z3l^*YXbK%J|)4pyAIt*NNd3tT?m# zTkhH9@YpjG$z*r~T6XO5O4?RW(lzK=gX2NObH?lP3S(JZ*9YN-*{9qb71#YLwyty$ zk?_jEm2TdE?8nlSl##uS)okEjUu@%el$c`osCIaxuyMF{VEdP7+Lb6t%tn%;^L#}s ziJ^-vhifJ|{rQQ8*^FtM{CI-4h#8Jo21UghG&hp76yt&Z?6X_HTOTuvFlIG+8^b&} z{?+9_!HUFhEcw<*$xj8NtWJvE_bs90=a)IMi?|C&0C4o`M1zsHF6WJdh1al- zZWd76GZE&Za)fE>>Meqa10hag0L@R#Ar#$@CGG$%t=C`FZC;5MG+_x#xg?;h5WJ$H zX83aG`qoT6Wj&IwN~OXu);!H|f^oU>f?O$b@r_wt5`WNOgu9{A+gFkyYr}b`%l#C~ zJn2i;?9rJs#bM|#ao5myB-$-R`SiTSio3UmRMAn&7W zC!3ioMbBbrDC|2|96Sb`O(68M?T02aFwSbT0YNJs&9AL{(Hp!HeF^egH+3VGW4X;} z@!Jphb`GENM6M!QWZ0jjGY=(8;^6V6WVHnTnJeGrJFM+ad5_Yqnwt#45nmn_t#9?d z`z)(f;TdwP$$8cOvO1=&sz^e>M9IIHr;XYt{dFkQx{aHSG=8gla_UY7( z$CKfI)8(-vRGpeyDQW-Otjk5TJw+&t+Aav`Z9h71kXw8OJna_;cUxo13(nq>Fk2bZ z$L>Ux(qq-xx_b-lFJvoVHK+5`;RUXm7B7Qs*ig~iZYuSYd)K}O+r)A2+M3}`qwn6NSxDdqp=Ca_s^#&e5|ZKRtV5Q`h!TD% zN&tUrIY*G9gN&zMtfD`hc_AC=yUP77)BgF8cI9E}Xzo_hj7uNcLVx5D{SVQOxB|er zn`guUN<4MZ&-!E%YB>;I^x-6D5wz?7!MEvjlX?ir1PpM)i!m0qh}M9JR>dnSDtaB+ zbiGt^&;bzVoi!Hb@?}IKSM|>^qK4sWdy!0})X<9k%Hy{?A&d%ZW4)JZ{C?9_(-?-I zd1f&}xx+ZB-m?BMxoC1880+zs%pCJ1W-w;bq1aLy6E_j^x;bRl-L}}A#<4v3Dx!3~ zf2KYMw3`Iw4XM%E6R7y4Bwo5ZllUt{iAbgqWA7AIR|MYHiX3d>-SCGd;DO#L4U?B zXJ*?kf^+eSIZI{h&oJ!yn8`${5LR2Ahi;?oU-~@a+iOV=WVsSYUC)=6cd0|vekkyj zyMCanGGUE!U_~Ia9GJBuBt{jjJe1_SW`%B8+*8v>6Isj%$-_^Z0lG;uZn@TPb+Vsb z7t!V#0d!<5df%37V0~Fc_%Jx8*4XI1iLfRo$cu2=?yXXR{cEqG=A46~PO&_sx$(dh z_t1jOhll{E8eq(XBSvA99HuOPKaBgG}R%R|JN5gEP(R!{L$saPsn`i6vc??N{h07X@e_Dm3=Fa zS&^8!UY%u6yY}$YzV^%bp2DQZ5be#hHeqE#f{8T2!6b6)ZHpbgC64Z;od-zVk+;YO$Z@m#oRM)u3e#*{UCMoX=10sta8#*EkH2K6<{lM!dhsn0#oMS-YZKVw?L(_L1#ecd2t z&_(|h&<;{l^x)_D{92A@sjsv$s`osGn!=#O*$n8&?VsZB_zA}ydn?k>1!R3x{MJ-F zloMb|swW}ZPgZgCIc=3?*43phHmBY~IFun*d}SE-BY%w2QJ~O{WHLS@Ta@JL-&_Vd za6B0jF-j*E0F{{K0DDlln)+Z3`Edlc)RZHgw4CU`_3ZUgSY#dD_f|SUG{6u@m*8@`Za^~ zrjZwVd*dw+5CEqa)r~zMC0}EUC%(3>JBlpw=Qcjuk>r&xxFHpP=cc1D(k_Ee0kG9q`)A_*=JQ5@C#JW&cE*3tob=oK8qd`^$U7muogc_kXXHFH5Nx4Q^RM#++>Z@2v`vAV_D{1Kc;o$Zz@P9hd zC?@~x0F`$hXJu8{8eq!tWV}FvllY@7GJwlktd}nNv zh$JuQE+u@9O3>NU%3%mwd`d*C1&GEg?xcF%SZR0*n4F-lQeeL-WP0lVxX=N5U)bn! zCIpXC@eP|YcTQc!qDmP!yr@tx?-XC>(84RTylyTzGukh;$S1T+VKPuLDDy^dp73Dn zehO(J3u`Ag)eX8Jf_!_w5S8~no(hKr#&U|dZNxn?OpSj!A4=%wSu38;S}I*FK_dlz^SNONyoVwg_$ae<

9ScRt#f%5{u({~X`T#8P#7~NcC^S7eaft{q8v;=*q5$Cu%&s6wUQZ_qXOAW6? z`pSu$IwVKca@6=c^>V}R)Smu08KU^$fSF*$^GvUmu$J`^qG&h#Lt~&gE_t@vb5$4w zo@omN9ppH$OH2j<8;i4vAP8pP&p*v_3Vxo|e&6(h+z9#7az!WZcqOE1*vk{ED-30x zLUUAxa?lb!$GF-nWjK)|%wCG4(=k0;EzjeC`k-dc&TL7l@H3;5d%MV}M<$*9Ct(ak zF$+9;ojEC~5wsdB{^vG25IiY|d&soxJ$9Yk+Aw~HdT4d^`~*4dKZj=j3$tGG`q1V7 z^kd+iSxnlAG`R^E;vhp=5V?)*{2ke5=*5ig;!IbQ=X!{(NC_!yXL>}1pIPY z^ecA3)KTY?bGYY$0V2f=}4X~3j&+YMg+*dh} ziJ@{UL|A*!;M>86Kf;m!GVjT1qq{&q#LYi`P{7y8^w0&GP2cG!pGAp=aEraEY812B zvHR3_5+dPp7Ja-*TD__~`Wv1m2`m>o;%m}9flr!=6_e<}`RqmO-QqAxoQN;P){fQ}jz5LKtGE6Gz5wi8 zP!BN^807#I%XaiW0xBEEeOD$cb=CH7fdIJ*F5qO^aW_KVC-`y7$C+h-?I}wETZNS|F|QXMGUGceAi{nT(qsWsaVNVZ^!>O-=N;zt8EsBn z$yxs$c`@(cwHib=cb0P$PdO|J(JBdw-A>>$NOfqjt&C@&{B5 z8=239efpp(Nuq`6D!}7#GYoOyLLWWqN{*qW|E<1nXwc7&&^O|MH7 zZbzQBe7MA(2^wkJ5XW)PzzW4cB%Rrhn2MU^^H%V10~Bl^v~4&4KhAeRh08g?3#>1+ zYssTz-k@Ap>EgdpO&w~?1`Kj2FBL9o27AGxRjT9t6$Q;}Pu<UJ`hIyerEha`3+V z@GqTz&5fvcy7%4Yy?mGU8QzC38b9m_RuusFH}XG=bGPWG*Nrce7a29YmpjqBk$#Dx zBB7)~O!c~bbe>=DP2%*qA&~Nq@A>VbN?+ssEVzR(apQqb2*PJdrM>=>=@=czddSA>m)pGlsYS({E9SgBs;bCnB_f&}_@M&*v>2`}b{q22lglCsTVq5}qWn zQ^vnml23hbUH0>5_Y+$w@zS`qSv&Ua{CAFtFsI;tlhVOu1A;i~s7ixjd-cJV;0u>4 zC+u={v1V5`wm4~YI54AyhTh3bfEn+@n2`<~pX91`x@FLwy}%B&0|ixA$kZ6#^{0Mb z{?D@J7Fe}U#%HFOF22XUjXq38dm(4Cqt`XQJnjQAAr6uoqcT8Ex-lqTE{wFL);!HP@hK9?Tz)47i}P3;OFrQM=tU zl4r7O)aChjcSW@TM|4MRTcKT}*EQ$fRA#~}CCj~>hZ!>}uVpJ1t#wcBaOZex z71AK+4Pd>HFyb3jC#H~pvS-0vbjsv&b&go(Im=lgDhK@p=9niGfd{F7j|*Ia)sFvha6uf$Fbt!-s~ zDknv+Nj}hL&#~alXZ7q|6In7Q#hu7oRHoUDz2k}`bLdmY!6WPeoqdGzUCh&*8H&Mt zM|U(b3%Y#cG*fT7m-DS*deG7}Tz$P)pgqOB3HfdBwItUgeNjF{C=BU4$sJUwLCMye z!xvWue(;?=T=-K%*`&cupCVEbf&XZ5J7%CcIDT=jx#0|tIkP??N`B}Cs&!>{GY%WElt7{-t>J1gLX{B6d60#`$7cee*HI=H=ZrA3iaR8yJmTH zGA;E)XZT4Mo=uwIkvu#wUIK=XbJK;}GxF`9D%RJ$ZEl1ft_oU@4v^I~w`Lk;g=P91 z_C!9VQopuu)!0!L9LRSLvH3})$BB31s9cB^g;9uG*JUTG+GjfVGk#xAT{3<)!aRQ) zU2-J}NSoBTi^z{53>zP{l=iBs3d+-rN9Tw}z}Y$#OUyad#=HTK5opgMZ>YAJPd zi`>t3N7x%&jP)|t$-})S%l^xAbtB zk#8}94tyCf>+zd1Tr4FH!kVN>qiu_jStot(nV2sv_k5PtYGazFlaoDwGT8N#q?qmf z*X}k~7vLw`KgFsnXEO+@5I3H-Fw>b8JR#svaSxN$?R&z_UyL@)QG3?sr_}3%qlm_= z`zTM#*Mu*$fVBZprrs(xl$V{gd>zjLgGEgm^E+_r_?VB!Y-@rr(N@)%oA<-X-p8tp z(%~r;c(J_dzqD8jJN;F%9*f!BS&)h2}#NW!W(A3?W$YIrF!J z&aU!d(wp0BmYY+$hAY)_F0|%K0+x!h&$xrXM0c6ZP_x45Y!e~x8B?;$e9@KV+{*1v zCrP5`57r|Eaa&9c#K;0Nua2|cYizReUgCUYpX5_q)#Klyin zb2@TV&`f3*J1zPW04kC%&-^r}i5uQ)tPb3d8wTT3dI#1{MHSjF`fDKDc{RIBMT5uFKds}|shs`-cxU<`XqAX`Z{l-T<3v;?tegNJY9Fr<= zF#|OG)f&BP|EI%)N=Y9;It;r%o9wof4>CqL1Dm#&ptsrwDx{GF@7{kRe;dz#bc2nv z(%h^X>K0BG-Ql=XND6>y_Yva$f`wbD2IWZI=v7Bs{(?=^>3Aq$;FpMRYOE1nG=n?n zViz&kyVr!w^zDYT5qdd~dbTlU=WFd31}DRkK@+Wm=LbA9QIL@0i=DmF%Vi(G6amu25ZJrQL4avLaWZ#((&v4SWl3PVPU|48ZJ7(7qN>=x z%kQ-ujJSOq7YJ|7VXDK0^oxVQ`>k6{_!3VAx>4!;3$64@)c5v9vE-^_wSBR%0(nptPp4fj?2n z@v@twiG@MTOs~=Vj*85Ie9+Ip2|gc|Vt3cSkw?@Qh34s;x>Ry~#uF+r&Q`)Jr{x z*pE&^?L3|^Yfdn=yY?`T$8#~x1Asc5lw2{=kg%v$XMea75Q&+NnDL!VI+AXy2SeHu zZC7W!h&f7qEGp5ohc#)R{nnpeKAtQBK5gP%UIjy#h*J>bSBnp7l_uc^v1qEroSm2~ zLtLIrIv2Q=kq*=_qu1}i*%m4L$Gq++ut}!0pU+&(;aN**iXOrFR<9?>Z%)L$l>;qq z49>M@3Dk)Iuomq>?}xG-b2FMBL#E7f7b@mt1zA_LYiA=rssQVVu46*W94hK*BeFFr zTe6?{d(=I|uTbl}_vZJjfLkVgxVQhV5loI zMrm6;DJs7b`CarNSS6aqrTio1wIzqw|BnkbX){LQ+(*nuP1p*{HiU~+E7HaMz_Drj zNHUdk=^)*6Bt3O1GWtW8hpLz8zNR-_CSd*0bL^T~Dhcy3z@p)B*%LEOLmd@R)C`Bc z5I#b&W49TsF0{*e1UKM>q>5+wrk?I$%%)uf3j(or2zWHe>Hfgqcy#SZlwfa4dLrM{CWr@DGZ`O&FS+?ufqLk>3`$h9s7w^4yyx=g96(3kzHiyh#s6MUsJPj5Z;5Ncn z2)3?JUozuFoR9yA3|L1%QsA_ZS#(hU|0%a! zMl1gZXFOu!F0nK;6*gyUoH~iI3|s7)HLSY)F5r-DPF^0CxE|fZH;`!p=yipecCKqk|E7P?c})QT_)~7#7^DUcKPG4x9A*^@_SZ%HKv#ZQyHd2 zR`D33y~5;(qboXe0p4zO0pr)!q%;OI9mnPaKb${hqDLKcT|@qe%XPRT=Xe0L3F9u% z?5Me9OUe`|mtbK7-PK+p0HG`PRBUP(46=4;2A0k#YB$tIcMp%*`fPR`Y*w;)+io^db6h=q(ef-mlv~d>K~o>9-aa-Hh#i)RR++2?kDi?nO9DH2DR(np=q`o?L|PB+DKQ;-NKYwnQ)Pp?`O0ujn>N(jVEsl|2m`(E zoZzxe)R^y=ZPNG$L<8gD3A!tVdOxBik}&WB{9o$FNFdg?4cxeK?lP{29LyzQUVa% zk1P?B7$R(Nx)n@G*@Mrgx=PF}9anoUmCCE=Y&5Fbqtb^QjQ#QOJqfv>4+a(u>S;EJreE~%Bfn;0R zw+)Zbrxl3)Ted3oDNZm2pYtD{hC(~h>|&cYnn^QAUsNLT8M;dbuze?+^viSZhs?m) zy_B0km()e<3&1R=3L($)AmtWrmpeY?Hmz?efVB4jv~XbFMEo1)xqaR{h^O^U0Y^aK zKLdC~OBr_`S!S2G{yQcO;w@$2zAYCt;B%){QA;mmJkbFAR25x`+zoegpip-=cq)J~ zk@%L{I9P4I=l8etM@*}dw<24X^-)79GzcXz9#6AA#QY-K^(=sq}qI349t+ocU7z)chejC8q z}@I(uYlyu%;A!{j@fCwCrkECylsU0FLBB8>SbLf-w)QX;GG$a(?&mGs-rkv= zzgEG~JMbU3*y9B7(2?))rltyUzy@3Su$~6_IZ~66zi`33zA<5S)4QS55a>_kQ3qDz z?PoPquds|BNnX_Cjx`CO?-YzlcID@M zJQ=kB!Vt^;>`l9H5D2Eaww2r0=mh2>)3T)A{xbBtB#(fqldBouwtt)O0(R2 ztF&exSzXXVcn2ECSgWId8d%mtAL%rQ4<1#qI@>>r7l(neip4~ z%Y^=nl$1fq0B4`OBgJ|x5Bzo)G~tK*$wjFXsZ_4Y`EwGzV5XNx+^?L4&Oq@GKes9{?>u(!Mr-?XmKuRh!FO-}8ZT`q{N%Cq582x02okm8c`^cF66rXGcAR z-PN=sY5*tj(~6(yz!f_3Okg+}miDWvIvVO`5mJ}*Qmy-DE)zxs_*L;PGJn~ zum|Q)?;;~AN!QoM7$VDW_u>dg@LVmRYu{F{)Jf{cvxJ45am+c=bXHDvlyc7o`i?Xf z3tFhyc6um>#W~LcyN7I$a?N)G zRzCV>aTH7sidgDIAiv4|YHGqaUVC+PO7# zs~;GPZTj?cR}hx*DV=#rT?;>S=+NZ6Nl)z{oL$Qpa>67FoBC4bNb{*aC)^@f!p(Wc ziSQQ2c;*_5b>8_G&W;#I3HJ_vKP|-1aH1#iTUgX`3Pl{{(1-Nfwoe5KXP1rJ6O*vx zmQ|Mt%$eYrHqKAkvgT0Pxawfpu*if$3v4-9Hg0~V zY~1>ECF?b>t$WJa&5xIDU*1#JY<#4A@TNP;H;ljr1n>+|Q8Q{V9V^3KnH zwk*AOTiNr%3*~H?2>duvZ^`;>2)lu)BfHOgrs+tGJ|}>K%u(dLax8s3 zLZ=^c4k1p6=Z?^21v!M>y=&|6F!G$q3@{+49e`~fe<*0n+s9D+K8Dgwg`xB#jAN+2 zrovGAKNgtyIW8ti&Oz%`*m;QqAWnA5jC+%r@$MeM0KYkdcB!Qa;&k+ZKqW8sBMvsl zh-1NNFd=g;LuT2tI0W?fJn|8v?3^P4><`CB8{54|UXc9goU&w+7j8!~PV%w9@mi#) zEqA~^hn#_Yh?d=Lu3i9W8aZmFMyL-a6LJJE(194TR^lWXSO_EZVz-$*=P0Ab59aIhGc zyjrrTZ%Si#wrx}8L&D|BpAbeqzk=>aMK%xEhwHDuzRQz?E{rUFKjg;ZOWsLU7luwcN11#l2IC8*4}lMOY18!0)QK^P z;I&8MTf&ZzME~euQNu2*{$~LqySC~U>9zoZSjAssS1lgsqwPn-kTUYgQBUoWc>Oe_ zh&7Q`@ErebCj7KNyLfGk^vN=VQve*No-}oD{)#}%Ad~x)rwlC#A2#?T)eZmf8=Kg3w-q9h1z1s5*RiBQq zk8cQfgniSR9c9y+y=Bwbla;g|D(hEPa=p5e_%#PQY+Zk#Y}v50+lntBqlnq;|%u3kT zZ9Y_Spc3_+Wy7}JW!r=M%et-KD|fAWuzcWSx0Ul=RU2^b_%W>fuljgF6liWa?fgs2 z+poW=+_nC}vf-h}%bEwDC>y?3<+R0Tl!e{b>}$9q@Hw9D zil|r8e#UEFSKjr>PnD%#d$_DWw7=ZG=ZW&Ay%qa+lsggiie(jd@84aP@7__?@7q(h z9Xe1}fA72Hre$}RS6}t!GH(%mkl1mZ#)*1uxVg357oJxx`sn-1C%^tc+4#)z}x&VV-Yzu1{x3WkcqV{l4u zAhh!Q&O~Uig|Nry^kKp`$&i#E;D=V;56~`6{Llp8188})3&d!BQ%?9GfiDD}` z6s?|tGKw!=cKiZPAN$FMYlGHK=3X7nhOoDCUun#g|FIHwVd}^{!>MgQ`jxr`?U5ba zBO&Y@)Y%YrL>;NtW(bwj9Xaj|bQ!9PHZ1ocagL!Ph9QEpnM;FKH~mkTaYSx=lA{m( z5Q&pcdCm82IRd06PPccxIqCAq4!iOi--wa?k(K_gGa}gTv}qsxKs_3-*?k{^6FGL8 z+wy7~I_nPzXZtmCKn`+p$0A_6fYHKY12W@u16h%;{vm`tUw}te>@udEq@jOyEFt|! zpQES6N`EDm=PlSUr+CyoUG3`9-828(5t3!WBEB86#ooo1x8DZHX&q!}sLa|j_6oUY zSG0lS(?flwZ5Kzr$yO$Hqx>f#(ofQs>d2V2xNN>+hjK#4LWQ;xN8j}UH{t5SD+I#( z5T6|N;nab}C5ueH9hEjw{}zFuwNv_Xj?RlOuDKY=bpf;sKki9*l>ajMIPf+KJOF7SiMe#*sMv_Jg(!V+;F)J$AOW{l6DXoMiXW^q||k zMc_SKx%8n*)E}*cy^`A1m87pm+@CD#D>kfpy1U=7W`EhdVQ<;8X-E0u=Qo!(T=Cwr zXyLh4kX67+bbT1D`wO#i@;-mTxz!+iuzdD&>&mjbzf#t1dZMh~Qsr-}a<@HQ)>iU9 zws~J!x1|#Kd-s%e+jf+-TOY6YkC&w@A1rV8H*jZN^x}u<0C_brI@&&Oak=oaHV|K)euksD%FYH@vTWX8oqJu^RK0 zJ9m`Zc05+Tuy03)FIL>Xe`mSVZu_ps%gRdDH|^P7w(rU+lvR5kE00ye{x?7McWJ*aKl!^l z88q+_x*u1#{{7#*c--qq!C3g%L;JtR90HSNCIKtO29wS&je*e0Fj!noL?)3;5{x`q zt)pRb_kNo}-lm@pi8sOsBfPRO@o~5~1st_@hu!}Wd*946@i+iphcP*F{5h;GEY#V# zvRY4;bR1i|TAUR-;E@NMXY$D6T-%ib3k7yV5pduX*p;)e$d1`1;Mh1mmpJmUV=e95 z-uAZc$|=rs@5*Jji{mAZox_QURPI2vE z(#^411b_pkomxAYdk&rV5Km^puD>ad{+04ghs+re=Z)hX2QGgrnDz=u3}M&r%uRXU ze6~C6+9K)tI>|TsaFo4z){?cE`i?sC`i1<>$sA%I6jQFq?w5TgwEEGR9DT`Rm-k7{ z`BzV=VmpVg)*sa(}ZE8n6{Z#$RpHAvt-hjM$Re;@8 zcVsDt_EPt3hG+4`?zrQQu0Hj7q)s?L4pCBHWFuYlW|OrE+eTK}PJZf%ef6CHI;ul$ zz=mpff8EFL(}wDv-pWRv`b-~7JDg}p+kvvN1I9v#V-_qRggx_@*ARqTNO5vQxVjA3 z5$(%{i>L0{W;P=>TpdZ5v^fD0AKE=TfwYRF4ao?|N&5E0nRd|TkqhqSmvD;=v2iW* z8U0io{VY2ATep1kLZ8UruZk02L0@Y>j>zr`_dL4l@A{26#$V#F6+qeyFTlv%73s zx2q%V_bmHz`S6E6U#@ulTkB*ou^ZmOBujYhNOR6C=U@27^5#n1KmEClWy97=_`m#2 z8QXHOtl7N3jBTxieOtxWo%On_683uCx})6p)%|7jecvyixou5({nhU-i_Ub^XQC33 z*=~p1WV)~{JmaEr<=d_=AN|6rvh3dPmNj49U6wz%qbz%HXIcKW{gtpED67A5pse}k zfihOH?wk9{*6%)9Hh*(Z`Qqw(%iG`g@v`W=+Ic#nTy?LceQx(f`8o+#bjJ}8X}jzQ z`}~^k8E2OZ-udqGp;fEPonQY(S@UQm<4-(MmQ})j=iZ&=);*Q5?|!0uv0iVl*JV2& zFJpUZz3tmumfingx%Q(UFK1t5g;bfqF;N6~V$wf#UYT>%>&x5kxxIYhiATz^=k}G` zpWj!$^xWRE;`#k$&AuJwu|NM|`5%?A|MHhFCG5z0a5RKH4v>y*0JIZk--$9A7?cJ} z21H*TFaYF`Azcg&aMl?r$7Brd$Y`%qq0M825k`3Bz)rM;eXM!klw-~T^^DVDtoqH_ zoDSs0ap3qku8ELvNWBM_yx1lVkqK8iS2|gtw^PSKvI6g(b4mwJ0H>M*f=qE-fPBsi zXF!?6uO3`|Wt%L&#gi`?>c~Qb4FuQZMYcZgDlYxicbsuYZtdXH*L?_U zTfVlu@*zXs`T~-!d?D4+0>6kX&)EaV`ct;(OI}-_X#@A{6k;!rJogNeUfrk*WfPwM z>Z&c|uP)@>=HJGXjtpg2x8#x+9oRbIiJx+(ZM1{%JfpMv)RyX97Hkx;Va7#l#7Rfkmn=B9Tyfd8doMK*hnb+-`d%@9Q3^d4Kz0+5Fu@mEb>CK5^H&a^<_;TZ#L) z^Wr$Vzi8|Jw#_kk`Ar;uKl7@$mUn&OwsQN|zgbr8-c#<`v7_8k2jbElkC$b;ca~+9 zm@nN`iF+mQcU7#}x2J5{yQi#q`)Cs6nB|%4k;1N7NC|S9FB#7yFHRC5-U+93c18NIXc&DtWqHT652pY) zxRE&(I5SzC93pcwVJ5N)A?$212dB+n9kJVe8=2p;CG0`+d?hhczQ;n?Qx9#pdrl=d zUTNpc;s`w~D67{HJaLYdup2)}kB^0AK9yHrR=*9>?!qkKcrO)+G8WYZk`?lrymFti z4xxRXbU{0>D-SZ}8pKBkQdU=Uf8x6rPCtE6Skic=48oFMt5;;WsypHCwZ)L%K__{5 zE?wGoC~hD07BA^s`*o9Y$;WfZvsUkfwfQQ)w5bnB+|(Vp%9;M1^qzxf(g%JE4WH0e*$r2qPz^5&V};!7I{CrACqUMGKXg-2guCo1DTJ|APB{3OqHyrng-(i$jg58B0qJ(SLV8zrM_das zvFlEuCBOL7Y=Jp9vgJdztHopOFFnGZITcwJ*AaXRzS>kAiz&)RKc^?!*HJ=dk$rx@ z_~}RPouCN8H`kI)j#p~bpEwrJvct}PDTg>--%ziK6Jl&k0i{ho5m&1=e*bvr9*-&+a$?y|8G^$lwd zRFeKw#i5G&d+m<0aowY3>!wG`ihI6NK6T68JCUR`C8Nq$B5U+YjSdC?i=%uBB*Z@Ka2a@)qQmW|&zSk`=VZzbxx%JOgQ zDa*fE3H-M!d4J?!S^da?va&uK`}Ur);XAu3$^S8JpOn`%pHTjW9Q@Lo=Vo2@7h_`?AleyzYbVE1!Dk-m+}}6J_aB zRsOSk#&_1A-&?*^N&9V;w2!@TsI1%hXnFhxFO>iB6UMwcWq(^c;onx0JpsRizx&%4 z4g74;c66OP;S8(+#W^tWIWz{DPM<*_O)CpBNDy{MPmy%*|JX6}oWb}07&-m)tu8}o z|g=d_zIN~!Q4z_d%x=Gt{bjMEFi^!8d$AV2k zdfX$)u3z}V7rLDegvq^0oqb`q?6BJbK*SQq@0)JAsawS-AK~%Ec8&_iqCFDmC?_%} z9-Ag^bdbM$;f}E9i^vF=GRS8L930o^A`Qm@F|)%Do*{M7nNA^gey48a?b86Sg@?(QQoLt+u(?R-vI&#}`rd&b$EOCWLW)Qj1e=c3(z)*V7 zKl>0a&(wk6(htdr4he7L3n#Ci&hO-vv~AqPYwIU6QXi4ynmkWrq%NaBKiE~@W(P92 z8H4P<(=NuS@y|aNu8wZG<(4iycFh%eM$D7mD>3E+^Mz+Q-G#t=4(y3MFnq&_&U1}-dTo_1;ZlegVmzVPX-W$lXZm96V{m(7)=Z(jdYhmC9ZSL~}K zerMS*wxev`u%q0!eOKAA;ajyK?=F|W{+e?78OUin0qLYFy>_IIK`8@D=#dpi6Zd+NdzUKN$+%KuJCggX2 z1y?&>ca+`zqH^99Zz(r?ZfRM1?>EZWw|15l-`r7_*Xzpf>?aZH{^OrkQeLmek+8RDJ9vI)fCx7@VyqJ<@$y?d z2HW6>p(caDG1xuhAPqT~5LrX$#~opW5ndU@VF}50yz;51p6VuTq?U~^p@)Fa7Sgzy z;QhBMVnn7q6UXi_#|UZhgqSWI2GQ@bM#a^@$E!-Utj*YI!;ZO_G55%B;=z!u5;^C!N8LiN(@5QyfE15S`-qxw5T&8=2qgctqHzjuV2RZ##zR*rt3km#JUh7qCD@ zm!l!<LdNqEqv8Hx{W{1!uTr$Rqn3t=f&AgrH0_x*hHyWcPJEcr%mko*&;{oR+5IEmLj zACi}J{bxy&->tm-PMqoDw&88qiHug?)C;h)eei|N5ymO|Z(~ciHa0i4V>&QjB|W>| z&KrT9WzWsGvGMNNz&w>=B%Y+V*2z`c3Rc$ujUUD?i)lCaM^ zWl<&Ui|d1h#b1}%v7UR%ndPE$-%@V8_SUlWOAnN_DE`OsF^n_G+jkl(Xn%VZaMp$H9x4)~* zT|B0?OxjA+>O`oDOKNv zS6o@%e%r0(OOHO%?VR8J?A~%)CF-|6v$K4$l6EA0=`(xEvZr>Hdn#5vwWmDvqvy-I zZ-2Ag`^|5a|IdG~hW_t4U7W005h@Nkl_qN`9qH)_{7v90a;q$})SBmksiEW3YQig^g}p zM>rKvg)OJwY45xpw9QIf8$^nA-5;kQstuL_I{~cmhohB0cfz&Ww1e+J&OB1_MNkfCV z^E#)(bHh-+mzi&lZgcuKJ7rg|UTyqO^S!|iUXHR;xNGMfVTUwlqf&75e_rcQt)KW_ z;du>jMa3tNIV+6ml?eJ3(5zONaE)7is24H%YUoaMtXEK@EaU^y~8^T~8_$$$!f-V1iXv+S7; z?khllJ1t?ohx!bDw}L$-VJYIGXB*SBjdk0+h*Qj?l9239$J&+qaq?(6YAS|sz3mCw znjfM~#JWM#W5aE46@L-#ZLJfy+CGA&rd|}6G)rNe!o0L}#KoHp9>Z?5#1qR0Vh&>B z7GnP5%{X$R0M+$9XuB!bcRWVp%_-Cib+;x_cY7RlcP7wqcLI&~MA&x@qpdT5`Wu}% zm3akA*KS6P+wAF%iWd7UaAR{i$)x;t&0B~q$IqakRsNZJyzq4UP(L^Z&(H)a2SjiW zPM~~f9Od2#RQN?83)R6#sF2?^!70>^K0@`doZ|?*2=-F{FiLzulz4sc3`<@|#?UGc zj;Ff^r;AImVC_apLr9oYsB_J_A=KHKAS_;j4OyAUe$au6sZrFtdW13&>sNjlLB0rf zQuKTrMNdai@njT@PbYBW`6DzA`%rkT9ZTh(`o#GR{Qi$pQU8DbfB4rw$pdMKb5o>K zz`y3rm(Pd5MnQOEgRtSkwM~kKvkOZ2t?;5)*4-^1ZKD|TVt#>ZUi@L}-(nD1GR66^o1cBm2E zC7xjzp6?0%zp^gj|0w-gr*Pj1y@uDbQ&~Hf^sX@4Ot02KR(K8bWQ|Lw=hIKoaGH1evHqmXrEu)b zWO|ryx-1LLJ7GP{`k3~)i5+LE(Vyl1u6Lno4(6A$_`WB>ZfNt2Gu7Cz<{%Q{%*vZY zM@hm(U`BKFJ$f!?C9cMnjTy+hd;^UQLA19`p`~dOO%0Rg6#3@IXlWL=Rm6PT1lrq0 zRJRSHrL_yi#SJ)c=nN8Paa{xpgORggBSJQo6`mWe#D!S4W*?57Ekj*<0Ihc)p{e5u z>hC^8?d=KF-Vxz0#~u;wwH>2q>Kw(*fhk<=3ZSO#E)JZ`K%%im+)mxx#|4?YxO{!m2vBh(1hBHF8lr%*8> zVqS!M<%kd*LAB2>LS4lApdXh#jo5hL2x3y_htj|f0}Ay7NrU9om5ivl3y^;JC{DDs zp@N@z`{5C)o;^hI6A|b?ghaYf@$&hScIO-%9KJu;0HJ_pB}tbJ752JQ zOmnlDPXT++IhxMTYH@wKR@U^__dG`fxK2;M8LJy`F&xFaPSK;dVmLO0xFHX}Ez6M& z9_wa393f#`oRX(;ak@MOH%CV3GH#;zq;RA^cgD90c2@ja`LkTq9z_?|MDu#4fm8hG z&jtj4>-B6jam0asOq-^aF0W-{jiV1dr^251fejqmvTL-c--ihHuYs0@9iL0ETd|(g zV+v)KFFQ%B-$V{iFV}(U2>ZJPyV|f`ugC0+;kV7X3_=^Ong+Vs>CH0uROR6C>EONL zy|+6@^f^=P!n}`HdKK;RNPYx)!|OLm&DQ=8#g zw{yP{`oHV{jePflz0nhdr=lAtj$Fag#TyW7j0Q_%V|piZj*fE#dmd6#)???+ODHYv zKx@m0FfQW#ku+GQq`@R2yXBFH^oM9}8%JZaA1xxjZ{8h6OIsh#Wt3pavaKRuxjuru zNfHl7qNk_LyAhMP2TAf^!icQdJTI|o=5V$bcu`QByqZD zAu3+Vi@zCv!fediumwkJJgA-+N8OLlQU2@^iXV-OSQn8lg1z|J1d2thSBP69zZ;)T zq4kO6xxW`@q{3djW}Vmyl`gzNY;UyP5HJZ*BVx$ZR2DSX^SQ#%JZl~1SoY5LWsRj{+Mm!2;5h4{0f!D)T; zX9Ge7JFnC0Ir_jfTkUc8UcOJ6SHd*vG2y*&1pCa;I_hm`7s;0m{L)E7Lxa)D%QO)lGi_>{d#q>&HaA6OJ<5D=85!TBw9D_LYezKSZ90n1 zb?4lSRqe6^o5$QEM(bF1Vz5luk-+6U+~b4qMAoTVR=ig8OxTIe^s!@-b*iRK1v~T1 zylOeJo}@qT0jK#F6%`rj)pCE||H5}K*qdt~;%f6lwATAjR(u184rE|%DmQggp-a~- zfjJsZQ92h(m+qFEStt)y2X5UGf!-qGvThs=VyCWY67?;UsB0B*Dx$vm`UKi;Ls%J4S2~{y&tIXv1}DnhsXP znyzrc9tjlDobIf))Gp!XZu$*bPBHZPeADOn9c*16NVEaKV>Vzk92;M3z|fx!EscwL z&?)hH-P@pGS9{^GLC18nQDS$`=(6#{b&KD-V7KP!W3j8xtT2qOMW=`+EN4#LWJimx zKh*}GbxJ(1QyW&eL?e7kJfDr`=4M08^I6yYu|fT*^6KE=@L}-(kpI)_^1ZG5g%G}f z`5xvwR#EY;F6$>=!|MtE&*^FyUQaQr;+pk>+SIy@;V95~p7jpvMEcV%!zi@Nu#Cf6 zNAdlt?<#%?g96xkJ&!p{rPVEhky77 zBOOHZ&3GunnJ?Ch`o#>ck>~r5b{K}|8IITTH(mZ_8dSuqpWX+%dq=ch7`(bPJDx@Mv6A!=Hugop67KSIORN5(hTDcH02uAyfhbjNym$d?>FiVHWcqk3`z z)jvE&$&*RsKN>g0deM^!6g?hC*%J}!QzNJmxAny%w2lv>_-Z?L9Y2AT1@f?qeOHVd z(Omn^bjO?jYPE57#>wHshmEu` zZyZ(U&bDl9uu;WL>3EI}4r@n&Rj_lq{fQGNj4)h-_v>H(dOBY!*x6`cdT5L3V}nN< zPK49%fBW0t8fjvlIVwW9$q_f-d0V4J8!K!8IfDI7?CLWs)-XI~16O~uTzMb(-G6=F z_#APW2pgDh6YP9;ROoXg>(HS?#(m*4$MAfXN%(aR4i29e{!iP%_YK9m)-j(8jw(+i8dXtdrHBbtOL=r`K9xy69RvFRg;z8ct!Jm{#j~LJ|JHj!oV}y5Fr}Z>oKa z=DLSy@QB#07n(-V+#G7UJY)C&QgM3+@C?{o_>j7|fZw1Uq(~ zLT+9w8e9Bmzx5Ect>dT>p^^rEbJLf=70=BV%#MY4?u5@y45mH7 zjZ|q|R=2j}_af5&O$56+!fqAmR(|@x&IJDR%>+9e4h{}KKUgVhWlQSn>dbePK;cS3 z$*Jo6EEYG2;aY8qRk~K_XN7)$|M!1yj9%;aU|I32U}r^3K|`DT{yBT{DQelvPZ7>d zbF?z&4(}?&+3;Wkgx7HT9IvOiWg0oPT^kn+$LZplFQ$W2{V13@g1}Kz+F*l%n+LHo zf!A`1ApMwjhS5*g5RQbULF&sgHtG(f5!8+cQVgAg8fab^_U&jFxu$jF~ztx;Q6d`8aq3jI4gux=GhVR zHo?x09^OMfTO1+d^T7MYb8O(Ma1X!E!NKA4VioBM{rUgQ|4Y^j+{;Aw7_p-JqTIQ2 z#}M1$(4S~Hy4-(-%f9&kPx$WORBzU+fAJT8F)(70tJ$#86T~1$T{%D`VnQ3Icth>HbL^EEVWBtf7V0d<7YdymBPz2MTGvc&P zWydt@Mf$To@08$y`4#wwVuC0%GHm#7&YoBu`1Rv3T<~oUf=x^S~fl zhrGBwJc_#8_p$%v1uR;=POcSk&T0G%ucO`+>~1&W*R92YiV{@#2hi|j64fHgOQyz< zKRJRzj;=qE-y+=0A5NfVavZf|qi846SUgu)g!wC1nH>p|f9@X;!_EZeJxJ}_mU^Hgv&ozK-fM_@hU;S<@8z2{}Dthd26fh-@Ug`1PHaT0#-SU#hpqu;$?*XJ-ie;}29SW?25;up!BN_rCZX92`CmR*`3Yhw^>Q z4bRvq&G!)N7cOn0uqD<#N8X3-r^0J!i{g~_`5xfAgzp5ttNB~SH%HzXme+H%oC1^a zQUGgR!E3D0pDs7DvyQOqJC-BteBUq}MKJ4A+7Dk&#@RzmGX*|3fn|8svFt$Rd5*Aa zy>A_1r(mZrr$0Aw)w+>0$h1!83^5h#OrzSghGG7-LxP>=+F`+V!qcZu4ga^*d5WH6 zIkA5I<^+3)w+i+KPY?~YBO=;G0E=j^^NfpNe`L_o^cbzp57E#th@2b`Hg7$Egk+Ak zGgET2DJW^gKt!f%9u}l;#^K`ysA(85roh*?30EJX;rb6~yzyMzr)aqG7@lk5e`5@- z9g}FdHG<;0dpLMH4=D>)h)CsHd9fKI1;__1o)3;Z)G^7Jvt&K?oXAE|{TFM{ZXK>K{v7 zPsULuBE4jC6onJRD1JDKQW5NBBG#)PO`&db0@Z1nn&T~muyTXw*eFv}F^TCQq8 zR({W!g*nF$;M~3IXb{_XUp~duzyC}8!@o#F!V&Ba4h~;|aKX;+tZH4rsl6O+WChDG zgcYPVLafk_wm4F%;b@N{M%Rmn3wDO3%io*^uEL%T4fgnJn%Dr+il2g>#uY&w7RLd)5gQ>{b!3?^vDg&hVUh6)w~@oVu*{v@@CYh%N)-CUsgy6zr_G z=}*Bh*4sK)vsv^I{Q!82@(v^O<85}pW8(OCb~IBskhMQf`M zjZHnclwF0DYj+|pk-}X>E|ovmZySQs#GSAR>o)9uSNz`9| zEMonM2z3$ix8(QDadAcLwh!V;Su6G)K8GaZ6KNpm3$GP|4$-AZoZHe76^#`+Yclr3C3K`MvZ+MDzReF-hX` zp%dF8;-eE&5udsMF>a3di%>W6#XRysjuRm*$iuk!_)#2hZ9@5I0CmqFq3n@}^vQq- z^&kqy1M+(e6_3VIF*%0XsY$dTUMPa@ubO8CR-;Nal!DF_$r z))96Jc20fMb8INEGG|49{`~o$z#e{%Rr(jEn&3ikIkD7fP; z8#Qd0yiH(!4}x7AP?|2ypYo{%dnCNAJgqQ{3TBnR4VZ!=RU54Y7Z$dlz&S!_M)~U~)Ju{}D zbL5;ZcXOvG=QMfRrXAK7ypHehBS(%HQ^+~JUZ>r27kJK8qdnHEtmDGPIALAIHSoNS zaWS2&_gF7!hd0spFxQ$h4eVgx>@n7tgmov&it%wuyw=|o?ChANKW7gy9_|XysqETL z;C>#Q!N%|u>^c>no#k9^#=LO$oOTqjGn;GU8HSzhT!u#f_q{vyUcO7P)BPA|??a(j z4;SpF4wiKB_{gC=los7ZV_gufO%Ku5Op*Q+&5h5+CC@~#PobecfTk9qbqM8EcW~@f z9+s`#is-o6hREf2$7A9+63yuyiEzct!Th-^v3uun5h_M5zBFIIp+7xDWYTCaHq`2nyveA zv8Wmi_Xp5OQBI*QZh2o2WxZrr7(rQ&`1J%()8j{Le-K3tSF!EDaU}3Fa1`qtJr`k4 z@$PaZhlG5h{4*pYUgKq_M9Lhb%7e16vJCnCy(kKJQ6d7pcxn^{lR*@Upe}hhh9VK{ z6;osIJeow)<0-UHO~Lb^8^1lG8`rafr^}vx3Q>k*g-mFV zYmAB7rav1}dOfG`v&Ww2I89lnpR@9%m}e!=bA&c1cxjI#6KaFu*brbC3U(bWP_!bq ziet^Uo?||=K}Er?WylTAxX~Q_*|^hd>FTxWQm`{^OpA^-=snY8=HYt~?BV5Og^gY2 zhxbPF_a$N6cIbJQ{oBH67{YrSexJg5*YysckMMGL_~pXqf&Zuae&>=UzK19lx!#=q z45xyS?_k!E{H=bvQ6)E$q&>Qv;;!`()7{UU%~$8t*6+r zGQF;c~3oa~K89H&FFp5G5kst3;HSi9jzm1pBBk zX1d)0)b)*^b#M&Tw>pto=E3^ihY+92X&UAfd6(oPUgDx~H@>NEkc7n5Yp^{x7lk6! zOGg7Jqevg|pyWfLPP8}_5|;RJ3xl{|XAI9pbM6AJ~8$64d6O> zu7Q__Bq~l=z8GtAF5t|AYbbd(g5nqB$ax$jWQEMlYv|8L0V{jLMuHx*lH_@-VCRT<_y{}wd7k0u&&rY4^H}Zk z8@HOKaMU)_ZJokzO)uBxsbD8;P_c3kw?RLvVCR&0HUxN%4Lbcq4b#VTGLP)A(7Y;I z7FIvP(GzXtu&miI3J+rkM?TnSV!3MrQSX@^GY^hn|9BXGIB)Z}5^m>xeC+iOA4Yh& zJN$Ct^I#{~Iqmz!ix-B!wLXjlzN0C+bu^iRo#K`6=POsPyz#zMDA;)|!|C*RyZVFh zy~z$-h9%ZD>?-W7Vd=7-;V3xc;LI{z4rccbCJfKIlWAs0IOE~9tgp5H*4bj3Z#yX1 zxv^cieJfn9#${~`&y8DkSpqxu*}+f$@H+2p|Ht2pU^mVgOV0&!LpuIRM522I)~r5` ziy2MQ_zIw~d<@Ap4mEj43CsTeg{RBIKnQ!`4fYE|t~wYS)#HDXh(+M{TV+O@?f zA<X96@>%SgE!$&1Dhu^g-?7&j>g$GaKw zmEPv;|Bz`~J>5(_{vEj9nL}F5*nd(%H1WHa*WTE$w{L9OHj}znj;uHdG&J5{qTI7d zE_tXK+FQpbaP<6r#RajvB2G?QfHZ-Bbk@G>8?joj&0m*Lx`a*xT>wek9%b_7D!dQw*>S=hN!5J*FZ11y zFQ}!IkrfbRJtp_0A>vVteeMB2(vPb>b)r!mGwPX1Pt?oyOigyP%%2|fcg~Sii3xsX zHR5u7`i!#%lM^`^7xRD-r`9X|(n>$8ky<`94eJ{iTBZiwIwlLKSs<^)znipe&bz$O#C0OHWhd-2FSmU@K3Xu7Qn{C4))WCU zV1N4Y@3cx2gZEOq)YC-gI_3sz?NhC$B%6buBI{wk^N^?;LTwm34rs@*e*_NE<{S^3L{yL-}(I4He}TQ!tlcE!M6)9favP5waeP{ zD;<{Q<%eyDmIKvv0jG^pNJHGeX41S8h;u{v&r?;oyYh(-Ig3)){rd9|BfsFD4B8Fo zeU-4|pSUY=A7n!b?U@C3*--L{_<+XSh7 z^OD33`#By~_CAG4PI7-{#(kpFDg_GSR&G&;SXbfK&6A_6XJJDZ2UBNq_9v09I1};2NtMj zUHyQ6L2ZOvKg&|j+{!dRu^g+%d<2?;$eMtkit`e8lzI``q z8l)tNd`n14_n)3POgdgq8DnQhO^T%_I1+>G_It~dGAqtG`B zi}mc+%o7|EQuXBh#o^UEE*LB#fr!q`yv3CVH(roTRxEDu!7*>b7O@j@sVh|u&YeHT z5BY)5=la&@W6e5zyV{WmO)fFdY9*HO*^}AZyJV;ETdNc+^w+mGe)O`~uANz4mIm=f ztNsvJ{1>HC`0KiDPOZ)R614}ZrOVwMS&1rUy>s)TU&_m~h_Bdqg~=*J7BWf^OTP?r z-r~SRWEj%|ud**@0#b^DjQU%jYjzs$=uSt!qxMm;3LmU+`IbumLf+C6o;a`i^!m97 zV@Nt=Lu)?1O;A;%DJxM2T!x|!uiycxHOlP&@R~<;6SLm7xOV-0ws%imCAEi$41tdR=1w55C~85qe?wu9Y^x^y4<$#*a9f0MF@CfM{8g%0)=3GnFprZp(`k zH{}=*@n32Su))N{1hcxe&;yycm8Drm84wZ^}^Nna;`baXYxa3 zWEh~V*$}xq^NyERJ0a}H)AHk?W$30&nnOd4pqr7N-7hVB{P*V-NxtJw{H(pB=Z5~P zd#!5t;TJXxV+OD20%|xEP9Hyv;vdRRd>5mX9 z`B>$11+w-kM>jHn=4TPlwCS5V{0qkE_7TN_Q^rV+(+I4u?WzL!17=^;vb7eo(jNG_ zg*IC}lbz66)Mabv??);B=qkCrd31w2HuP?$!~&ei@x#~x8{JYR@KdW3ER*Ez#l?>L z`E!psWs_7<391JFn-0EMs)K@4 z5#7EaiXcF~DH?Shq~jW`viBH_lw#_504~2+yB^2@mC_4!r}PdH#PnVVlwZ9#Z0#xF z{cbCmT8ekC7ik(Z#7bIm>p#-hm;F@=;7H}&Q67pF+?`b^x=*MOZdwnM ze-B2KUvlIDL+XtE%VhkHKd;!^&9vG$AQFq)c)>msb2buu_C%|)eO z@r_;`=YqGX$8I&;-XaF}WioX8407Q~b#~Io zZq^>wDx)KR@TNpsr~>eumyxAwcT2{7It&;)u<7i+4)wFRLS710;l;~Qe8GIcziuvU zG@}Tgz7gzPOAihg`O^KyBh=tq!SDg?#sJHVCCBK&4C%4OW9gFjD-PdJRR%?>H8xWy zWajzQ=FWYBVsg*eDncYc^g!T?GXkHKG;xrf(qc^9;-m#Gc!Oclsd2a0A1wec zowke|b@`VpD1Ngsvj&ykss2>;pIVI<=jU^4ApPg=E4nei9x^ZYvOK*GHNNc`OA;rZ zEO#;k(^a`2_SXM}#yo&5t)wr&C?H(Re4jp^bD3_dyv`P?HE|*(JUM2SP`cd){8kWC z(e7ea?*vYjoh%NTjr--aBod~Y+R3M#PDDz6jiiYOY4#==XDIV&eLe;&a}A_G*(%y1 zIr0n(zil`U^+x~?dF7)bPkBHimwH@Yvp&XLMRGrjB)F)_iU~tnF%`MHA|DfAY+y1Z zc2suxmGQa3A(o$+s>gt1Ea+r}I58ILIs&D$pt~H@6IWl@koQ=kDxd zt_YHj=@Bh~>GK8-I6B1ojPnxJLi0<|C}z-0cNuShVBM;)r*oRgAKEdgOZRV*fX>Vm zY4)PWOi-`y6>G{F6|6y6;WLnevQu9ejX7ud>7ZQp>QrLf~5M` zjo7zBRW9a7jqX(2km2LL9N69S-;VN$mW598iRM*Su>n(3!-0xQV<*<1f|esBLU;)H zp7WU=*>$5tSil{kxk1`$yqwlOg8Wv;=>At?Ua&nEF6X1$u;}8{)Z@rWKZY^U&FYyZ z@rJ&yi^DRTmX0r^0o9-vk5=Nywu}^?22bn`cdV1J>bc8%N5`6@ z`J>3;`i(1>Ta=-G(M%s4dE_I6n38T(XHoj;UISZ|wyi|>TQe!78jvZsWqEJOYs=4C z$EQ`3TL|x+-9ccgP=domvx;^NcWD0@Mm1`hP})b6>qYf8-6YnrDlq@}W7gx@SkX%rCy5(8iH&Ez7DoGa)qnH*B{VgbP3Ae#h}CZg26OeEZG9 z{_hLp@1QO;cf96f1Z6*n02oR(@HBLMOr;MggD)DTSpGQ-a_ki*biFm|IVF_0U6A-) zLlu4Qf;omJy_jcwk;yM8jd49kw<3R> zPA=>Q=FKkZ_wah@{zqeE5a zbPeU2X0jvCE_r};TN~}72T>-WBLkYjXPFJ*l6CH?<{ro7VOQeki&>F8*7?82NZd-BWb*=Q`p0e zNVQ_cN3f?HktzsW(DwMlccFrCTM4#aYktpXt2C}1@NpbCia?IRy*TgUdb%O(_QSAu zG=VH*+8t}sOemC>ug0sffiL0I-VL=Bzt68#++-|yEb=a{?IC5@8=8YYJPYtqDg0{p z6W&Q*h3EG#QWkAy=F%|Fbv1RoG@7K&=Pb*&q`PZI85vKgli7Qoi8ZK-U7J$jsR&U{ z?2EI%{4QBb1KhYRfigY~V~91pDiXq~xk>b~4But;U{ka7{DNHPnMu!9rkB#5(iBu} zsv^DVqSmaxZ2Yg5a(B~_o5gB|gZwSlOI3M*L0xc`ZEHkaK~%wqCTl0hJ5g46nGhbk zR<@0eDJOZiQUPwcJGcrAp_+%mE4oxL|IXl!`yh)4u0CHO{4Kr@1O;vc05}kVJstny zRzZ;eFKogF$y3L}vMNPZ&(1ue3PAb3e{(J@Lk2#ZzkmN;-uusMpTdAPYPutrV&|3X zfz=%Ud=ur7J64|T8`V(J!JDZERpdL+++e|K4&;=mCz9Vi`q@1H_h-z?RL*En8n;4} z8@Bc)sLM4dfHGeZ=BmhBmcqkvm-GVa_p%6|1Ky;x4gQdk!C5e#-;O$s4}MvI3UW>= zzya48#u!<6?40sm^WaIj3_ZyuJXbrNAa1?$mXL&%Qz;oRL+R&7nx*9&!NDWA?j_(m zCm%XFDp5WWYap3t*PZ*G)DSkp`_ifstQm^=9aq6tJj!yoei&ELsoQ(OTnJhWbYndb zn9keQ|9IHJ6#J~T{%Gg+u-?-3tHAbG(Y-OW@2N%Y7;jY1CTpzj8R5eCnpNzHy_~w+ z*EOq?;v*yzzidg_QU0DQt=Qzz82U!Zg@~@`4B!)%rf=|O4|Nck*3twCE_iB!9#9K0 zvP^u}%d$IbCyHdmHJEb+I;znH@a@q+XFnU8hzoQ@_%nDcy*uot$gkVC{oSI10$y@H z3Jr6YUUWFNeNv&PDo>~C#AsTmocew<1+{;)$H6ujs+c?^@WZcv;z-QKvr{`VQNz;}X*gQX-)mU!{iUOCtZcjp>z4=u-Rm6ffY05keca)f_l2Vz>0hsM-DA zFV!VHET+dLD@ANR&wG3{w5;}jt!CQ6B^sZVph}-k7hr|ty}0Oir~9BH4}#Tt}^si_JIDmZ!6?0%+ME->qA_J@X1$j~bsjF>fU zGh0^9>YKUi%(K}$@sHnkbim!PsI1Ixjbs{KaE6{#@ZEk-0fg{K^@prDK%|Q(y-zCR zeRsu9?SRXM+D1oazs}53L$~Y#SAzRL@4gRua^qcXwpp-kw)%2Jy(!yh> z6%OFo$UoGNw1!gD_*B}qM5l1@q`z8vBRK9FEntT9;gJUcbFcSZrw6eEkGV$aK*rVi zvJt72^_I@jh;f2Wm^=vVnQsl{WL$db5RG=Pms|)bkS&^(HCtL-33O*(Q%DS0No(m_ zIA52Jgi!a6l6Fq)3(5bJEIGrTFZdjH)Ap2aYt$`;=9FY^TL%35V(ZkU?(^k>+cDyK zLH`j~Y&+fL-rgRVQt8|!=^)`ik^LR%LVhf+bg0&B!MKyiLw+V{uqat&r}RpX)Sqti z)sM-qIzOjXFoCoq=@kvDnY6hlHfb_RoD3-<2XQ$9v)p|z!~ioYx^G@*t|Zl#oU%y; z$A|wqBh`kEo~DDABI0eaK@eFM+C!8%%c=kl6cbWx;Ox!?)gx3i7<84@(o-|qsQ=(T z%*@7gmA12mazj6yd~Eom**0pOat^KgIN`?!+f;3uEzmqqF_xKQMYNV1-S6w#LuJ2N zZRr@pt3$}laGmhXPLZ^gs9w#zSn*Wgp{vaZ_GANDN619j~f!-Fp2!%3=0-C3DPM;s)=Xpl|vGea7b7 zZbH9&F~(t{b&Hr}qcubjFb=d|?Jd}@G_Ba0blGpT9-a=kom3HclCs`*OC-_7cjn5y{+Za8wj|BykGX<{yu5rE!#3*%@Y_b$q#xf^ zklKbqc~>B9fLuUR`MK2xQ7GziW9(wMzPHI_XHCI$+VVZxuX=FJbzWZI7`j$-2=iCR z|LcmdUV)sKU&@^G1n3vROCjLz4)R){XX~tTLH=D~K^kxJC|s^30r&|tst^FW>B0TU z^y~8vV1n!$u0%g(-YlK~tmyw(scg~d8>zsqEvu-4QpCQKurle^t1>P>G=_a_*!f4B!d*04j$U?3OV?Z zEsF|C(ZhM_Zs+k zBugfNAk^z~`st#z_hEb!E5p68*~0?pm)+QzDq7rFCx zG9hu}a+0r;Zr?U*e{?Z!KVr-=otk8*&HzzD8wUn(4nIO>Vw`;1i)o<}s{-M3cXkcm z$@J^tUb3tyA|85j_lOl0rq*ONzCHiL_(M3(nB+`(k_Jyd3ZiT`geJ4V9Q>^YFYPED z7j4e`$*RDJmrWB6%WqWTZG;?u7{l77q@>)#x^EDG>$|(OItGQevlrz3DG|}pNbFzo z!v?UKegxK5e_TM@TrAsg8GZ6~CdF`(+^_2G z2pCWA7FkcC*MDk|O!>diyqbBzP`Yh}Xlo*7*oh3E@V@HWmFunDen>cbE4W4uM&+!? zhNWz`?%&rV{bCAMX$j7K@}h|cRYHpP2HE19?yu$sLQ-H?JzTnU+p3d&cFiEqeyr^S ze|2Uaa1mS!Tgwu5z?%&QE2+B-V>!;VZ&UG8QV+=!=-^IVZ5IUxLdeF5iBxa+Z|ReX zC@T!vhEw&2w-Yu5a4o}L1`=!q9+T-(Sxo||=TdP!9-XXus zjdHOkov56WIZhu*09t4i{beN1M{XuMcan_saIyQSUQaWEg%2}t7v>2}rZjJom3vBw zRVqt6mIU=z#neiS+|VD&4Tgv&r{uPw5hi`w)eKUTY@VhON6CH&wep-s#l}HNH9GEV z_bpwmwyZ>N3I22B&M|!aF5u&2uUdu`;?4Q0xc%1GXRoGHqU@BpJ)9!`n)RPlDyU{<)8A3?X-&G=5!wGyo=;)Y<1h75C8S^ zOgMPw!nv*Itb74PB0`9WZ+Gej&InoTd@ujQe$rw;D3|XyyixN9@Q7KMt^mubOlzb= z`cI~0YV|@n7@$GJx(X^P;UU1hBJ;CdG+#;(?zo-|O#clY4)U7sbg6E35RCF}4KVn; z^zSOBTCVL8;G9;>bA&$aq`*whT%=>_+xT@hF+cX!#~|@6&mSfJPY2><SkW;vUs%6Y`qBgtoD;h4iNDE(wP`C{RCg@ zLKDY>KYj6xE&w5^S?5$PO}Gm$XU4wprz_ug>w=nfbN_Y<3dpGUY=R3{R~0D-+sBN{ zZcK(cbrqQlOK(>b6zHSm%p@M9m>Wg{+^Lju=E|Hl5|QCDAXc_)~0;i%ET@EpC~N`B$upb8d^efc`3C9j|9p~C_n)` zaIxDX%-_f1^^PU94{k9%{N;Xcs)NmGsLE*fBZ%GVTE#lCv{T|%RJgn*GkbKz(?h)u z>bReaeA;<$vA)gXr z4Ls;|=4^WZ$HM&!4M&!$jI*%l!MP|f}{U2$=YZ_+STbn{% zRZn_qRPQ&-;4A z2OatlCTmck@!Lk8jJn2;#&vc_mo5Dj#D*CIR56YPC>GZmldU-f_+sPqL9RU~fvomlO(aFc8D(0}>W~M$v6UFvwNGj9RnjcTH<& z?|^$KVa-S5KSfHgOIqztob1gGJ2Lvk#hbt$gCk>R~yUK~Xq|_(F226P@uBKkECfB1 z7U=&6oTtrQ7Jb;+sXneYIg?jO+rE=%B=^v4s)rsPI^%Wt74sX;5Q-;9g5Zqu7@cXI zgS_b;en+$VlX5=n6SW9@Jmhe6d}vzCZWx{lS$5EHDr6o!iYHT{bc*PA|Ic0pw*?swJUB}raY@h!!1oJh`in{&FQ>yiHlNV!4xeXO3C@Nicg;gmXz6X%IdRKxN>U1 zmgHTzpvG9C5~~<8z$7)%a{Z@&0cIKc)?^}C*5dG%V`02?EfZKfWvcD%p!|n|J{dbz z)9Set`YbV{x!Hs=1hN9b^Lo|v%I`p^v3N;Wrvvx$(Vc!CttlQdWyxw7c;$KSbOW*f z(Xe%f=k&xVh3Y@4V)c{+^}S!q^6$c_VmHA?7a!?=heuazfYd8#j#ydt1_WmHI@K}~ zZ6!c^CEBxh=#E3JPf<1%Y&b{zUJoEyDmhqkn&;Q{PX&C+tGRo1G-bpCiUn( zo~(Gv+A0pRITI_mA%zj#Km=~~E?lZyN?qF097Uc&gL7zl&I)Lbm@i3VKM2%WJA7+; zEWAgL8yeTh;j-3ncG<|vTlj$&+hg3+sJ;yMB>iR3auTZdVNJD?r?aDTpmUKE=M-Kl zfs7Myl6FiGC=h3490agad@(MT>!}sl_~x6-o7vg8!yMu1mOlBL9N4+W#q#jf5bu;* zn7wXgmcpe9)f&S)fl;7pBxm~w_Pk}o^5TA za5+zfSP>;nk>+W8y#BKvP!ldcZ-M)^qX_2fhO`7wO{+g!yUriPJ5a-VbM?Z%ud4N) zKV6rHa~^1S+0{G<0+6$iG{USLvgjLR#@@>sMK^?fL@aCYa@So-~lWvjHz)++~i=jTuHh>_9?kDK6B z(b+1lZGOS>tDsgMO?%ezyZr*GMb7lH*N21bQ}hX@9`kXH!M`h<3F;#w{1bC*yP-~$ zj)D5)*4n$`5$sm82jReja0(g!1ODDemPOo$ zu#EZ66gY*%7iy9Sh`CoB_}ypxtxZ3IY5C&H-IVdeTU?rhSqt51T^wKEzU}5VP7tW) zxTk*lMo=Cz5mnMlF`*hEVjl)PLHc&l9Q+VwGFFze=jKSIHd5qzIQN&+HJajObF&?0 z*CgI>i3`9on?LVK<2g^`w~k2kFJLs3-}5v`aGH8Sbw&`~RSMYXTOiy1-o|Z3{?Pk7 z!~UOreHL=UQ3Y!QwN&DmOY)vEFx;$gw1^MI|)=)7FMdH5d5!x+-D5z3>Rb zN;43+^@U@l%MVPH{W5um5?W;LGaysU6vB%F0 z-KHlp;pMnj<@d{HoL;@+Dp=jeitbLHz3M6&zGlbk9PMHszSm5JX!leR==B^WZw>Su zFRsK&HB{{(!ml1|G8tp#Vo`2}1K|vYRrx(!%s$&% zj&mycDmL+2^v?z02*d*xsTHTdGAQac#UuLXwuE%Q!CF_g-YsYi&HfjJpg?YP#mCp+ zw@af}zI24`_ydaDAtLw?pYD2Yx7O`9Vvl;53}@ak37!vUyvv3&8gxVM^ z6xBHvx7f|kvS|%GBZ;u|TJv&h=V0S<|48Pd@BF+K!9mVkYP|Jq&CMb_oUfhixd#LV zjPYq5)*fgC*qyGSX(#dRooT)ZitGR<_Yd&s0_yMFIxQi{&i&< zX3tKDQMU@dj~mhd*LK29M1-F)lGk2e6p%D7^8UxxsOIh*Y03xSoA;KyQdR5KkfOoW zGQp3riKpW!z%J9Jrs?9Mj^rZry6k>HVZAb^gdNwuwdmwoCG0x# z>tp{ID?+}PNqwHlVv5?@r@#Iuy(7Mww@^<8^scbH+fVWtL7(cQ({pB$*WnQ|Rv?xJ zRRMu;hQbx82^quU7AwurGlz*p8;=LyZ3rOGIv=DOmQ@ z_;ks)mL!onhU-Rxo(IP?y3i;Q0hBq+q&mwWwOEsMSF0Kf$yXoxL=)bSwN?q8UO!-! zAPQ>=-^w&0k^ot2#Bq&}8&?g0RwXsY z`^e6J64UaR~iT&^1@`PVE^)gfQ6IuH(KI=v1ab>aa+l-DjYQ39`830E9NN48Y z=<~Kko^Kz%G1pRdiE!V*i)YZp?jc?#m#(V;Q+wb&Npa^D+&8tCj^(++R9}K+?9VSW z;^nkS3Ic^fQA)=BLGPr7Uh4r$w6Z}_Dwa(a{io!D;zcjHH`sM8NKeP|Zd{G$0XTB>MV5iC+;1hfHVw|wCcfw!MHAaD{@-f4xnpC^| zUhwrc>m63PVKhrETq=kob5FKI1JF%cgRzG0vS5ShiO5*fWrSuXkxv2(C z?Xgvx_u%Ym{8Tx}!k9Jg=J^>a1b1sQQ|YID#ei*vl%|s*fSn7MeVtO8=(Iw!63ZiE zro+PcW8l+S=H4x>@H3J1I)jo9M~1>GAVFZ(>+whY3a6{r@c_GV_>A!C_8C2e#GpF zLzL$znizFvdCujn%o5m@7;gm>(j0Np!5)FFA}p+88u@40UO8N3Zpxb*C-fD1{wEe? zN+3qD*gZ}~lhm(h35+5|*}kCT#!23+fL^=5EzJ&NWX*K`7|+T;H_)@QfBBvKQVi#) z`7I#f9`>gQ!ahDw<6}=DU7g?-+?FY&K5vLF2y)E)eFTV~pN|+}-k4=6w4eJQ&cLU8 z9J&~8CWd9Il!wIZkWVbUb!hc*1L{)2Hbj=!ZCa3Dxn^4(*=VPg6_o{G-J&FY=DJhF znwo4>{{W&0!{jb(C+wX4H|ND;r{&V5>MhBvk8oe>T#%n+E{3<;Z$w$6ai#OrVr&$G#BhCX`!jnhK{F~$-k;TlR3>G0u0C*7_0 z#u)fD%?%1|-bp+L=FRA|F3%XXj57>|m|Y$-%}# zbc}Mk7mpfed3vz_$9s!?*rt)!n0EHO0IbZr)%J61>-uhd)&lL;%YHv>V5wltTCjc- zvpDX3Kd%&7#N_8G15K|L+${4>>s#L7CLLW~FzxMJ8uMxz0RCS49;tr8W)@83dj|lK zu}^BT|1vd;HOqAD$wUnLM4WS8yy(GgWk3;E*|Vi_|FM{z{cZq2@68aL4ibwDiHVx4 zj_Z*)&bbvalx-sB^n`IiVDjw0ThiuE?WGB!y|s6=|Hv-TO319Pb^6S#+Q*FOR%1bK zE%7mN;||SU9IJ@>GLt%!nE)R+k_aR>kf5yfjjwg0I2#ALfPx!_K_s|sT@8t6xX=Nr`>rmfk z?K2j_UauQStW%?mFv5!+O0R#-vj7t;1g>^h8^(l1GRsMh|kdzMM@W4w@Ch0=FF*-@SX{PKAKQ6nOiq+;%x^vIT?7 z9)JrUEIlca^qLQ&j7IKV@x_!BmD@5I{zs>@zG;pMv$^eZ)N?Mr;Vy#TohIWP?EjV2 zxq~9svgb<)LYw%Qax5df)ML&le&P6c$Wf?S4=*F=`KPin6sMG3_p4w31o@t%7A5U8 z{l^c6h7Df29$%i83t4tX&{ZF9?v=te;wGLojis#iEhSSXx?6)7&x7a1yS-)Rxbdr_ z2OS#9Cwq&tsYP@FCQW%!b2<(lb3*GN^iP~()afh8xJF(=;ma0Mi{cnBCAZ8w~ zbGUr0(m=WB>$%D`oX;O_Up))F%eIzaPf>)@>o0j1F3P0&Q;(kGGY0(p#jW^Lf>m3F zP3p6}7Gy#H0kLdQw*THH8%xcusEb*%AH?9shfc76BC z-SQ|AzN2DYvK+^Zd4xj3*F3Lb@<-vO`C>gL=dWR11EADwU-%^U0jAjE=0S@h$}#J!lS~;4weu_QmYB|mzK~t+qy~~wHlfLLyV+rZd8RE zNXw&&X1%IUmT&Jz9sV16qF>v~&z72RKuO0jLYnBV8^4Xm6S5IO_XGIa<4L)0745RG zPpOe22&vZsEW68CYOQ+efrU6Omdyj6h>1BK^`&#QjMllngk;NmR^7Z(KE+IjaT}<1 zc|cLZhai(xyMqgj!7m=_`bSs3oSq9z$Gu&ck|+ruJVq%}g(kE)`qrw)gcY>O9)-L$ z#`kudLz&@N2&_*s#XopI zv30-4DUiP=4Y~HP?M+$|VK?&0adJ{$9u+U^I&g)5-mFnP8=e1zoV=$?&&}HA5>O~V z#oN}s_LCElo$cF$4I#q_rab8ni6pap!^q&Ji!DRT5{DzDaNo9l}(Om|I~tOzzxmVj?_irJ9m&vmB>rt zCFwqydA^9h+-1sF?c2WBP4*H|2b7gE^Ch7+$+e6awar>}rn)iDOqPJV2}-Pecyq*=X}%4XJt6Y=zQP2K63k>btk84(OCA4x)g)^*W4UNL)_yE zFS&4^(fBju)_la>jm1X$K>bghWAlj7Z~|E+jmJZ7Ammju#$1&0ciOt;0$iSM^tpAp z0-%slnX1~_!g)k&l$BCVPsA#WGODYjgYBS30GFqAEO%lw4}9oYMRVZx9_2JmZcIPo zzwnJ<&x~inIGiNdZ3m`D;tiV~zh5QqV)i&-e&p8>HWQIAUd`MP0eS{LH4Yz1;xb6@ z+0=N3mi(b~Rb*|aSVuE7_UG(P&7p9K+KDE+rBS4}aCyT0J5Oz?DSBAyZtCfLn}vbA zW$KJ#F2g4hY{A;Ip3D0n75`#kMsJ`QH!aEQvS%)Wy>a+h*{4>Z-a2BVj>D7Zq+FGy27^H zEYR;kN1LWewq9o)e*0%2PAi}AG>8sH7vL8&w~tvmKBkcF~qscF1`5ORh2lt~tPkrbP6B1}Q=DTJpikcYDI{@zuho+m|cK?27 zD^)A&D>!)HOi|9m#h3Wol#G!q#1;U=0Hl zzW{CP(!n=R2C&n%o~v$bX#bemcroW)jg71tmMOpoljiG(Fmd)ym!W-kam}hPspXtr ze*WZrDb>jM65Tu*>)&$+B|N5KK5k7uyakA{k_wXI`t)_<-^wCUb4iJ>SSGn$ym;O~ zXQx}PDAz;_%u|mGEh0)TxC|U$I&7k9;uDs1U_g5-@&18!7B|u>N>3qiGk7zCiNCRl zRu+5RI8fM@ENfR%&6p>a{B7P5y{Zv|Ds8AMMj1`bFC;>!4U#k2`m=1qw0CX}k*X z?s*E*$N~xMV0FfKd0eM`4eot@o$qBw43oEzvl;9+Zv)(9LbU<)-$W4Nz?KzxF332F zZkrk;EMeXG@Yyr=Os=8!;g0W$>=&@`gYaE#LE*oxuD|c@A02^$R}GG5-^M#We6I8W z7t|=X_$mrkT1v)Ydy>++=JB$4N0$Nd&JQ{8%lt4smlFki20?;tp3rlK=y|X{06AYX zJg&Pu=;0D1+?}j}B^7+-&A!^x8#rZ@bZ+r-)h0N55)+JsDJGh)<6S`7 zFt;ne_j^{fFq`}`3+rl?bU4LDL;A?4?&D(*UzVk_$;*cLRTYt=6sotoYi4t+3ELwcA zVR8jT7`q8oX62~)Hrbdn=-!?9Ybzon(QJ$Wdf5AxRFjM1s6k*V>u7T za0%8+2D-Vi*Qgnbo11x%NaQhDVzIj#d9jMBjlW#(A#p7Ao)da5+#Gn&Hs`->Zr1X~ zm97GHfB$at$75Rf>ycLT`0! zQy@BrDV6d}H)>BZI>p2Z3JBtY^aK6SprE*I>Uqd!+sdk{0&>Ia^3W^gwn@2RQDN=1 z7dFmwyB@!z%RGh3%s4hhJ!Pn?0q4%2iXs=&^E-f`7GBine6FC}j-L~%F&&v|LAlh% zn8mpAa86luEdukGX=)=_Ge z_T-u~Eko(wNh^o&(CKMFeB#m(-ho;8>U4~~Ih}q?J2`Nk=+ez5&|<)lPX3p4L5+VD zItmp-3;A)Pzt|HGU5=IhqTSvJZMf4Nnx$^#Em|Yj5AI&vRD~T{E;ahpnw{t8=P;JJ zjhATQnq1ggycRs&zk2WgGtO5I{`DW#FCTU+$;hMl@|%Owey0^D!Y?`cXE!;ykMUjf z#Pu+7ci+2r`QI2R46^;KrkUT~s=Pd6!BoR>#W{k8pvb;COn?JGM#jc=VP1uZ!Ao)y zNs*f7U3=dnKUe!uwPiXsYQ&u)+{gt9MTETt@|kew_gJ7XDJ)-f1sC|h<) z7$l5!tVQ-1*_s(lvhT*A=Ut!Q_xC){{l^%tYwpW+zt5cWI`?^<*Lgea@n*|6w!In% z3V-A<)t}nf1cYW|tW@$NJZF^R@$ILd|HOnHZk}-~DJdnVq)4}NwoKAjN>%R1=8>;c zU#09b3jm7hr=LVCn@~>~9}*?{@VVdkV<&{XkQ6^>}& ze`K?d@v(wsSOot{XwfaYV*ie|di1Mc2R(#pfFIzfTuN6?Lj9xuUt z0jbNj8>Mbo8?%TAa&9@Zv9-*tC`@PQBKcB^u>C2*h>w{ZQqFMRh9A?WgITg}f8|CT z>Xgu&V*ipPr#qYYZQsm5GhyIj=BzKPm^$G=M}vt5O5p$Lhw`BIV40cap3e~*-pj^Y zpb$P?F|(B}MQIW3VKl>Urn2NG%rf};##g-miSqPpSEWRGR2A%;k0NSt+8@JknCRZB zBj@2fZ75W3XLr}w%*>_no`%LN+@TsfAT`;Ev*_61-UqU$FgB>18nZGvpCeIRN{U`l zQL&dCI*=iuK3ykdSuW18oZa!H#$>p{Rz7BWMZ=h#vBS9TrpH+jmugB$|62!n_N>T2@hMJKqJbVv7{Lot7t{Sz|i&l7J94<548u2#5YBtLp z&WKun+s?d7p9`@LQU2ZB4gJhp!u-3~hgzE+@@#>ZVRTN|T}mwIlF^iW)LW#aWmS1k zgKBI_dUxWA0(d9|+6=)Lc^W?GDg^p{s!-P)Pfta-IUoi845pBJo67+yx{!JzRY8u>lQ>bLZx|iUR{${yS}E=R}-6GIM1+-=je0Nuqa?4 z9`AQYna8-Y?3U^|E3^t<8QxuP_TNRXJ%L>ru1V#&E3AtRRKLtSXHlh}F@pRf?s zN5knGH0U#{kQCaYgYyu7VOU+@x^FVS{5Qm|GoJm~*mAR*P_;7~-sKq^ekvc5b~SC7 zdH|Q1Zu9qYDOTQ&``tRnWZcQeDgZ&Xb2LQEO$5X|k=Jn&J%aoK+LMnS){H;Uw8VgK zShP|J550cZi~>s6x+4KT-R#e++msT%x^{`Q|JJ=rX82l#_|>KsHAxn5K@)ZDw8am& z2Rd4MT)SB>D)*qNCmNaK9)-&Cn1YvYSVdZu$tEuZG2PCMEN4Uw86Ol3D5be!mIF$DX)k!%zkiKuo+2PCk2li>olO`<)TW8{(E6P-mS;T49o+CC`jUw~M z4`TwGG0#?!txE^HW0}kFWw`Qh`F=pU=NUHk$2RCuAeHC)Qbuw5HLZ!4K1C7ipxjx@ zO<{R+o(olOhB>cKo;ET$v^F6;nP_}B81i#eW;|_@O0(^{q&?DUMMZs5T-FBv{Fq#I z{qIavfr9Pi^jC~}+cjS1#R+Egq1j4^&pMPY@89~gT z-}i?YEpR{_XDD(8K44;Ub#Vq(;07#+uYsWW)xhq-EAH4Vb05@8vLx4#=pE9!!L2I^ zg|*Jz%P_vk8aQ;ZV3-!*8H+- zq=b1%pWQhztzh;${Mu+E1rErW%Q;g>4KXReOXy~}H)!j7Cx5zHW4u<=v{}6uccHI; zfEqQ%mwy0d(e0X}Y>iCRe7#%h8QPlC;ri+$x_i}^Z(WPwU6KzeZ{aUIF1`C24jpz= zl){{vJ&|VpIP*$`K>l6-1(GGu4>Z#-F03hKl1H)8+kU&mkr6kNW1w(~y))L==i_L=R#`_}&~n_Tn31PH54(wo_g9yF?; zRg_njKt$k^hccEq*VQG?tvvLN(WxdZ+i#3XV11g;DSq-d4pfHYUA08o3~(DP-7E#x zP<;p*zW-hs{C*Qr-Zq|;bL^^+^Ck%(7)2{x$f8CV_c9^k(X~=$QGR^}{OAhqeelQt z54?5boETZBd%vI&MlA#=bX^jx>g0cP;Pbwv@$&}GyNSwGZk^GhV9y{PiV24E8RRCF zkm>N6CbaJ_oKH`n^i;eVrQfC1sGt;8iQ->02ryRI&_*Id9oE8RbSa zi1Q@&`Z`_Hrk%ZOO{@A8B6&){Gv0JI;Bfl(R)7>NG1hgl(VPybuj!5{KAu;b%`VIh z#7XYedu%XNFS!Js!~CrG4?VjcyqMbuEmRQ|{gU-``1UW3p`b*T{iO7*)xW*Ip~~=R&&FjPUWS?^n|Ssj%<`j|LB8V)>}5x^ZaJ)O z3Ct$-DU=W)sqP3rv{ym#wWTSt(Kt@)F)%v8;zTf;`UPlJ2+&LW3WAcXu1B=)%=5k9 zH~`lbo}Z!1yJHFP#d`t-ug3fgmIx|hWK3>w7oc-mDmYoDxOI@!G0t@7arHtLNB)9x zRLlH%%au)C4G-RXLFA>`b8of#GaIb?&P>>n8xmHD63^x2+69)Mo;L0+VN>2D&ce7w zcsV;?Z=aLVZS7}8t#mupGkFi0JtjGjwQI@N!6ms?M2bR2Gj4a%s<(ATMBw6YlG+_X8y{BMLr^ zbA2)}u!y;>Oox9M2(Zw|<*o8jGz5#^*6GP;Fm&@u;TBXKpsUTh>IN-a-qv@D3ueV)H8C7=kWc>9ppUXY^Mt9iX!vaKCWgg!M><0fzclYsu3y>oRmd9sm)4Pc!$z=_e}fdDkl`qHKbK zf?%~*0uwtHQSU$*5k^(Fqo6NU#xklglAykeS50wEGf6u$6urNKQ1JaiI?$9ql~I5* zP%xd!k=d%Jl;vvLQT{0NLGwKuty4RSCg$CuLm z_6vhMl6Lq(n~`dZhlc_le2}neYLeQcZVtxQ318A4q(`(b zN?~JZ<)79p4@!3rPF}Y>bqxnL9q8#%Pj2 zn2N}?8|19Wn(E_|=AX_wt6JfQ(x)AJeT7}v^um%n_X-v-`Cai4UKY@HQv^yq3WF`R zpX)%P(1SM9wH;^RX6r9vql$fFYuavxLSMI^SH>8i6eJjIXz#FuK7+mxV@%Z?ufO`L z7jpRuppCnQ;<7{bwJf=M9pPEh>wVwdqYU~UQq4bh70^q{z0kIZghP?F3o9B{nQV!3 zjHN3a(0rAo>39VW9!4YzTq^oB5#$`Fg^x2d>}zHc_NrJC69dLl3>ia#ZV#=~exBc2 zlXs*f=?`$;Ve~YR4%0Z#>O(3cejqMdgEHJ_k&VT-KofU8dv@wxa6PAgaOkY^{5dxL z&8v9%o={uryb5FHxNxZeb_kOCKwooN3$$*MBc~n}4BBEziU68KF)J}*#U2Y0C?C_mf@lbE%kg(91W4K1Q39^E~JT~yZnVx^~V5niI7 z`DN2ci4q+fzXg;9%f7lQtol6(K#s3WUrSEK&{6h2=RF898~+Ck^!qkcNkDxLg2-*% zWI>;pqtLzoJh9z%?02dEJTdKo!gle^TRk|#i|;J?3RUC`ARqaMDsOs4fDD+#ri4oE z*Q>M|jUHVc&UK&g3o@RRjv7N-r_nV>V!|~@b7~V%y2535dG8k-D7!;`!C|W#4I;&F z=-BKNk2K#P`~?EPKjrFgia-pxPf%XyKcwTEk`Hfr#|BKMd`5Qw);+1Bj@F_8w}q6` zQW^&nB@djQ@%w!dgET2%zFAE&f%--dMsxz!g7qvv(ZH$~3%4|=UI5B#D%ZM~F7rwn zkRbK6;K{n{t$6uo)Q5$-p^qMrjMcDlfm(v6J)vk%sX4;cG3H|S1LP#G&A3=|df?(&gKVM4t zv|e%uhg(&-wabV9uvY)w9BUN1FDZj~Ujrqpb+Z?ZNP|6}|M`Z2=gRZC((`mbH@vz7 zQ!?h*Dg+cWU=8p?u0MQ|@d3om0nJw))Q+K2V#id;-zN}>x(0mLZ$B8j&;OF<*RNm6 z{nzUr-t<^y<$8ZZ%RNl^9#10p&x(lTgX3hTvLr>8zo98CKjeX0 z4M8xqj^J4>?(;VsigFYsY`*3!F(cxaJc9G*UfU*=%MQBQ@eH|kR=Ixn8&_;)t5!Ce zkD^cCpZV_zE4fysB$It z*Sw-OsVcFqXQYy)G5#?hSDFWUUVXiKs?|?>1y@UbSlPiR z)03pBW)lT#qCUhA8Cbq_>Ha!LW^j8MtV>Q^cKOVlDq2~DXJWYR&%??y8#``lVI=cg ztAKhZr{Ww<7`t}oIA~GrK!BXW_Qr&w@z!Y4OVFlJ5=A*Xn{*giOfE-RNbaH5foQNc zwf?b#LJ$w*#%vBUV(z5UA~n6_an2uRU`XJliWBCA8&)&!^AtrdHMd)g-7omZ5uN80 zl&T76?*#-Hy#9l&*!&lu{`VpmTxlymlCpb914Z%Mk99C?wCK!YArN7y4ve3&*9|nK zqdd&WZy^b zRRHRN!bi&!#?+L?U>3Q&*6_N+d^WBO&AMoJ{^Hx=y_8JuPZ}_c><`9w3E}WX z{LfUlAO{Y-G%p`uD3@f>6Jbno_5r@osTMNX>|85ggUnr^FJ)x}1ytZYSH>VEv{=xwrX*~qPTW^D3LAfL9^edO^ES#6gNDEg~f#R`KN zdi-I=9ae-*`ge^cq5kv*#W15Naya#&_2qG(C@J}9VHINFraHOyBQk4|8FDQmH+#%W zNqMSpz=P)S=-G`QC{AY;Kauqfjn~7mnhWo!_d=ghLG;poCgYZAw7aZ98`h65wloHx za!hh7$3#WPR}C$&HN;suy?Ozs6={MxOv|@pzYE*rHY`^`5?ANI%1n z>NZDC*sbszI0XEtR@a&twn;GZqI>v6BF(op*dB+7msLM6?6qxi)GKWT+K+LepUa*S z*eD=7cBBOEhO!^tMeZL0oy~<8X2mAEw43(*j}-q!;s5gzo3xl4VRb7@Q6d}8Uc5yc z*X9{2HnjfpsWSq`ZIubXA}_coM5pzrPY@hB57b`M;#wLkM9@?5h${Yc8bD(>9;U75 zzaJmYhMHFmf1h_g;8IID(+gOTSH+LWi>q>Q?y9Rvz~g;xSy-H(dx0?@4xNhVF>Ba7 z=UrtTDg0Aw$%P>rJm_;u(a+MEC$6dLF%dhULE>h=gWCdx;I3Ztlr59Cb!MlOyh@o4!HCbLs6*kbHNVaLi zSR+37Hzx=EO)VRoSWlcFp0(`S!QB?>Jq1xMM)9Ry(O;6H0}7yDs8r$1?k}VM1oY}o zc4+rO-u5!b$)jfsp9@!~i-hRdqCdf3gi_2gtaqz&`b>HB6`aU0&#yqtKf|Ab zev{~fj7NYm9c8@HA*Uk0R<%aM;m2p8f0+RrzqJjtWTE*A@zc*&;K(Iv-$JZ`W=?!z zuE0jBx#IL{|baZ3zA*=VL<`i!B4k6GgTl9UMj5532xF-o3V@1A|L(fT zg_AQYt|sq`w3Ai0+p8r56WGqLBS9dyO6f9BHZQcoRnEF${tt(&U-Re%k_6V0$etU| zc_r0&?@SKxKWpXCTprLMDE#T;dl9Dks#(|i*_=y9ZKN{jwcvsM;`JU#mv*P*Qm=wS zvwJ11OuU^GhYo0R_Od*5Z#A?7B&r_2Lt9EZL@FZdcjZsJx}L88@1Xwu3>LI`SYx5x zysf;*%km8b^-)qG6u2bGLSTiW?@6kAzXH19zpTTK1Zk=lG)}|mg$~h1zU`dRb0Pg~ zsL+yXHc}*bHX3uLmJYWu_oo-zoYQr8qj<83mB8n7pC6=EAHRN(_-U7UQuDXU;B+%f ztm^mvT|}#~jZ1pSIYkHS#e`j1H#LHSp$PM~^=*F4Q;b!UA9uC^x7OgSVyY9UX3)7j zxRh3jpY;pp6P%~pWwGm2#-|^oEdy6{CwDb08An4`zVpf01bS&e$l6{$HYDR?a5$n& z?NP!3-pmisl_(Vuvwpe51l1v-ZhYIhl?;f`&3Bj)+%sgYs$NaqO3?#mnnG2p`^0{H ze;oGzb6AEQM258|c4u;fRzr_n-rB2*ddaa4*+fx>dRXF%J(HIFhnr|&FZ=z3`0Pd* zU#fc1G$l%uZ!+PO72qd%Jc|du0;Uky&>h}da?fkR?1p4|e1OMRka6%XD{^s3AcVl$ zEo5S5QrV{;qg9yO-sClVLNfE+qegy`D-7t4J&{K2@js(A*IdTF^9m01WoYVR^snK4 z4PN{ZNzwnpqx*Lx40K!Bd~C25F0I?8sA?VR6Q%OPfN>1!)?Lsm!>r12XWFATS)%ZC zN1V47h_i_7?J;q%HJ)V05U3ttyc5Zr*B8pH()h~CoXF7b(&j}7C{o}Cib`lr&0K-C zX0(d>YQ|v2P6lj$IRvJvWRR)c?Q@Il;9_mWp7Q>s z%mXm&W(w`Wh|5QA{sJ|o&MI~s=~}Ieyi?g=4?bEynIYc=ZoqU|pi0_rE}Z2ZAJ`Z7 z)=$C2EMW}IEy8S<=S&R+ZD@t`bFFD-B+HJ(2N#$@f-4$dN1lo_`LMyHEHP^(;t&4O z`PamLTvi6LmN_^~3gw{vSfT~AL_%+YHLW0f#q{P(;Q2o7mLxhppGJC88Zeug4>K44 z^^TOm_ty?XrMF8x%k(lM{Nl47c%p(eqt%7auQ=2H)Lgio-nLh8B3~qOGMXgbP6H

*;^-{$1WQHF#?V{{AL%Ax- z@3{9_G1DLL8%70HRnseiXMF;0SYBJxcy%Jl{!1fqA-1_wWAW1`hk_SM^kgaXaR9Zc-3>G?(KH{fNd)*XJ` z*+Nr|^fv0kDS^2Rr~zFG#!RkXEE$mdk2s> z&||wA3e30| z1$u7T+4_8xB8MHpxH%wP$|IwXkhxP!XlJSEH#eq{zY zdskXS5ZYJ-uj28JQ}J}JpPR9BV^dydemCI8nYOoO&v4V#JzRh}Ft%wXE{l1imxs#K z_W~oN9-S}>MGgCwmfC+wiBy=kj2}zkPrrX`d=saApqDQ?FHBBRy&aVktJTlqef`)- zuw7q=oG`kTSVr$~P)JIJ)+G}Xz7aI#`JQBDhEJ%j*0i^s6LOd<555NJq*a%U*K0o- zi%&4M?)sLxU=CjP$s-#bsKA@c&NJ`NbuR0d0hk(<9y@*K9W_yMN3yr4HpBZm^NwT& zuT=&*DqH9453!awJAF%pH^#x$A*x+brW(hM17tyfWh*m>{a2p!JTjUW$3BkY)O@pH zG2+P_8zUm`6^9j4DYa|8F*Kl^Wb)P*enr9o9FV(^>HF3)u-`>zgQ6IL5N)5&L zw<(C?Pe&Of(<;s@@+1>N)7AHj&I;(KC3q_5n!e56LN=KF1pgr51*MqR_w*AmCt4O> z%kC83*U>54^=!njUtQ5et;VAWx5-IRZ@!`{5D6WHGRDkU{Fg0xHg1F5&@DmVIEi2E;7 z4}EU7s`sBKq{p8|%rSe9V^l5x!u5!%A_GrhcFuwx^sLS^)Trcc4z;h(SLNtZQOTy5 z+=$$%^6;|6sVZtC&_u5b>_*wA_^Y6;3Ql>Pk(V@$vXaqn?hY0Y-r8gImC?NQO(5N| zui<@D1P+d71T8Fqe^&d>VyY5ovswe(x5H5y$8oqNO zmcw#t>?K((RlpgL$-x2RZv7)0@e z`$uHC`3-teeN#)l@@#=MU#8;0w1jWJi;em1=C5Nvjdp%pYP8PnW5!RW_+Bk*XK9`M zms;U6;h)s}{*NDY08ZXG z^&WMOBj&Vm!YJuA`Jl7alfTe%yHs_E`JEfLo1w3OTOTV|g``erhw3Ah9RcSS`e_sU zT{~WxygfpT=Sl^&E5D5hOeX3D=sQVOzDAjRWx3Ll+B8BmNHJtC>-w+-i9SJ zo42ogTGvGwEJ)xXpL}}oYHa8;Ro**J@Bw`0pO@4njJjPPjMFbEdliFucjc;d9(fnq zl08Z8n?^w&-4n~;QAbpwKO3*Hp;Xp-mT>%*i+X74 z!}wQf&yZL6{O?nqZt)=&Ek6MlWo-tQE>(RyA+VosW&3WFyC8qXhbP&U-6@wb$c=ui zd%(ZzM2pZpPF8m`h#qDb4aK3bZ2S*!q1zQl>YUvypzkchC*>?S>#;x8JpZ#iFw0udD}*DbLxnQN7n`JWl|kFCleVWlS8`gtzRw0DIY%!~w^=_s zh9xg;Y6I7516^Fvn;K{ReD8%LMFKpj`Vc!P+ta5IVIi>CIu3z`w{LuwkA4 z0G`!89+gN$@=1rx`@R+ZF_)Tj>XEv~I(%8k7euR@pV}GQ5FW+#*3LQ3K!~vON2I1TQH`wUw@tGK&6V zyAgIb;@*KhXr4uiLgdPPR~6=pU?u)m@E0lN^WBNGAQmb1)fu9lPp9HDy4gE<*lz?U z-(f0GT%h_;D_6&K=BdITZ-zJEA5CqiUwxzy3h&jYlS~pF%^G1$g--hB@Kc4R_U1H| zi)hKa0X9Wi%HEhRb{I<(GHySI=RbUN!Z^}A@}F;v49vGJ>q9sWqRcX#Kd}j{f&y2`P@!UJ;XrmD6m+Jn)RCZJRc?QnVc=%=x zqq@hC3I(#BiKpz_16d`SfI(OGjD~BW`(JP4hhB-xX$*bSc!+=5RbmtTvXCEJR3w6b za|uc_^7_h9UZf)xeZg8M#ca?C_2)dA;rzn&Slx1%Qe2@IV2z=RiB~yXXJ~-}MGZpEp%y9!uymU^mh$D=h{KDQZjBBSYJpE~)aX zX{_aF-8H@}SG53_6&|+D#o8@i(Qhw`O}dx0s~}{bN4#o~Z8_K&H7;!Eu$dR4qkF!( z$aZ+BC81@2ghj8OgT982%`h&ngV8mS` z^wuU?cqvt%iw3A!V56M<|FTcHs#B7Mt{si z?_MP7+0Ex+SQUBP@I1gfAh{Pl*TlTB@RR8vLS{Z~rTL4#O_L*9q z$0Q34QQ7Q^a}btR4s1BdpuO_P1=KGMzM;JprytpJwm8bv-{N@}a`e2?;?pHvdhQ;i z#LN|R9C*1-%F&^#`D)D7DrX{Gwd~94v0n^9?+nVhu=2G@$cfadznL}&r4!+pr%k#uq1)<+MWxOo|zW`cNR6kDHenIW5U6UU&8K2-B z(jPjRG$1ycze`@HP%yVisLyo;z1};vi+i2T#3t1)I=dn-b$+Gn0QXXt3Bf}TwuvH_ zI7cc7V@`A62X%PspVTMu{E_kC-|=}QT;O6)iAAEGdM%Uj^Dj>HNQTu^@)y6ov7esN z@U)Yj-WOU?#K9T_XueUWD$_ISIYE{|Ak+gx)UG0n{ZgOrvvro2+2dVzsHr12q4fIz z2^&sD=pn!NkvmF%{C=Qb~dcCxNk&aSB;^? zGwgJ>C=@EtGh|(iosLh~g8i|JS{Qg^o0%__7&6@wpP$h=_;+VqVAeG!4$NlH{hH6p zz;63M3&bF^GCIYsHH;V76(FC;`&1rEKg0}dY1Isy{BL7InIpR*^Z?v5H2?2EoZKNl z#g?r4IAqe<(?EZ}FTXAR!0UcNo7(_grCHlknjDj{rj6OeqF6)Q-53cS_T8D4>l-6) z_a+vZa~O@o@cq()dPhCGZ1{UnNtq#Q24gq(@C*i`&T`NF8|Ez&N3C96Iu3hQQ*9oW zpFyrqs->e;%J9|Js2|jQTWmvC9TD{x73wz`ki1IBfY2nXp_HZg%mRgWJ$y6#K4_2X zkn4i_pNx#unD{a&`>wD11vRNS7oX-9*8|HztM>D>2b-Ne%2P&%T?$o{WY3<3_^+e< z6Lj>p&O_Qart2v5{oT772)ekYNhEoc!y)CNxW1*%Y^+IM;>3f2 zZ*=ES+)_{``+%-Rc@I{CSw@VXVp+<1YdpBfgwJ-(DC_`$jh&&(J$wf3Tr%%G<>g9Y zpqpbp^=<4cFu81Qiuyjf^QdnXN0!8)(GZs3G*h6qsw@nv}BRH*iXB-Lw(6UUJPO$6^9l(7qw=w>Gxw zJy*Q+YJ3D_kT3I!(fud{MxsC=BV?P0`&x!#F=!Ou-?(?Y#?2Yf;b6tPe1 zJWSY&Gl3I`-xp6N6D>S9Cd@0%Y6rIn>O8vRQr3-8ECZ(YGZ}9EAdTUQP0Q*w7}0&v zdZ-7drJGQ(rQEQd!@8zFo{4|hD&p{M) z>#9-dQOuQAs7Z#yU8iqdvx`lsnYS9}MD?bBXOAm6* zM9*4op@vPkVGf*?n5{hnK{7Q|Vh)-wP-LFnQU2m*gl8`k0ihinG2?j^1*nIRV(B6;s6MeG z;PC3LLf>qdLw>0M}&9VUNwN2;Z;FnEdA_<3G0~`Ne8(JrPJ$2YU_43Ll zHMHla?0mu#{dj$b+ufVO+7C}%dp-oO|24kNzS2{Nk)tjYHT2SKo1gO2>>ox?9`fT) zxFnsP;eWPi>Ac%_mFFW^#_$han`Fj~>LCofzc>hc>2wF#bIFG=q1pn)Wgs0=)-`$< zfaK-+^;^XT^OCr>Ew8fo%b>61mWjZAbWZH;50mWeqCfp#&b)feh-}Y{t6BbWFkR4S z!kCt*aCyAu#WW4qd>BJjxtYL*7MYG?bza^Bo1m%;b;zI&iJALMJon_|)1+21WKHTR zVlFnxw)5?6JxdfaMA~F~>@&-IBYwbDS zc%Gm@?m1TrK$Uk2?Y$n1hg=lfkOeqKWLn$VEvb=8(>r>8 zZIBBHSxkAEZ|q9>Ppk(iVL$-O%>Ns_On{cYnEkU5k^iFMqS@xcV2xDYo1O6!2r{?z zjXhw-YXh<}Yirs7yXWt(-jz#-B`E#jOpK&zw~SEU`EfSM-la8_XU@y)7edxKG$40;YTHEpC(viC<`LdM{$t62DJEBjQ#78< zuI~%MtPMs5wNXier4@;|xM9AYqEWji*K#DJdRJ;MJeq5do1v#rc$gPE5t8jy)4->7I=OdNtzJn|4qpobW4xQI5f4ORX5$D|rW4g0keSbS|zh#O& zqXcMZv3HjGy9dxi_P-CL2H#hzBpn$6rUlr{|Ew$`_ak6F{@2-|ZDX)5N!5H# z0r%x8I+Ges*(T!f-PgR#O&=1jE(681XF2!bhPNn{b-N9@RFTC2pZPVJEWLWm<>amP zsXsa_Gz7*|6R+Gbw0RMA*ISsKXgNBok(WKxq+J7OeQ(=?R(3scCxKk*&?}eSKX-rb zIBK<_gaS7tb7-i;=~Ey7K!doltfZt6RljUW`QVv42R3h+I%D&vlFt>RC}53|Cz1k# zv+2w^Dp2OP@(zj6{+QPi$_)#zt6Dw2_cpFqn)VG|Nr{4nzWxQ3@bK>TR*WmkGFD20MfljnYzzWN&jh>4LC}{Idum22HV#;ZJ^`&pa-&8s8 z;PiPBI?7%(V5(iY>RW22Wr10F`0ATH{7+Bj?z%VP6%rw#`e!dMlBKS>HUahkTaa0B zc&pBaJM;$NPcSbO>J*Xt?eEtgc-DVC0q|6bz*KrPuMG`)FBnwp{%Jq|Jd_j%eAI>m z?W=udU{fn)Rh>pT8!!-$nEUec>gSYUAO{DJUU)9O2H}tGMDJfL|8F}>4~q=Yj`TL? z60ehoPSiA+uY1V!pNYgxi74mhc|`xX-8+IZraD~DvyN1i_d`lhJo2AHzc(aCb#ceOsMIO0><54 z$@zdW+l$fk*SP)31hycO3gz@#Q7Clug2|v1OM&`8ZurEax!&D-n6=_(H!O0HV;Epm zyQ4zA2d|7&P8U)lJB7~+m1aAPmW6G@#d5M3?c{`MLmPW~L6t~0BFb2IaY-R(os z61`-Une;!q_ol9A*AQl@Q$7={2+fe83RU4zB+Hn#rMPP5i2f1KZ}mfWc*#ghXKe Z@M`n~u(1t@fzJW!uC@WBO4Byt{{eHw>tFx? literal 0 HcmV?d00001 diff --git a/test_deployments/blackbox-exporter.nomad b/test_deployments/blackbox-exporter.nomad new file mode 100644 index 0000000..5d7adc3 --- /dev/null +++ b/test_deployments/blackbox-exporter.nomad @@ -0,0 +1,107 @@ +job "blackbox-exporter" { + datacenters = ["nomadder1"] + type = "service" + + group "blackbox-group" { + count = 1 + volume "ca_cert" { + type = "host" + source = "ca_cert" + read_only = true + } + volume "cert_consul" { + type = "host" + source = "cert_consul" + read_only = true + } + network { + mode = "bridge" + port "http" { + to = 9115 + } + } + task "blackbox-task" { + driver = "docker" + volume_mount { + # OS certifate folders + # "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. + #"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 + #"/etc/ssl/ca-bundle.pem", // OpenSUSE + #"/etc/pki/tls/cacert.pem", // OpenELEC + #"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 + #"/etc/ssl/cert.pem", // Alpine Linux + volume = "ca_cert" + # the server searches in the /CAs path at that specified directory. + # Do not change the sub folder name CAs + destination = "/etc/opt/certs/ca" + } + volume_mount { + volume = "cert_consul" + destination = "/etc/opt/certs/client" + } + config { + image = "prom/blackbox-exporter:v0.24.0" + ports = ["http"] + args = ["--config.file","/config/blackbox.yaml"] + volumes = [ + "local/blackbox.yaml:/config/blackbox.yaml", + ] + } +#Default config from https://raw.githubusercontent.com/prometheus/blackbox_exporter/master/blackbox.yml +#Example config https://github.com/prometheus/blackbox_exporter/blob/master/example.yml +# http_integrations module is for validating that the grafana agents scrapping config +# all the listed integrations must exists + template { + right_delimiter = "++" + left_delimiter = "++" + data = < "/etc/gitlab/trusted-certs/cluster-ca.crt"} +gitlab_rails['omniauth_providers'] = [ +{ + name: "openid_connect", # do not change this parameter + label: "Keycloak", # optional label for login button, defaults to "Openid Connect" + args: { + name: "openid_connect", + scope: ["openid", "profile", "email"], + response_type: "code", + issuer: "https://security.cloud.private/realms/nomadder", + client_auth_method: "query", + discovery: true, + uid_field: "preferred_username", + pkce: true, + client_options: { + identifier: "gitlab", + secret: "yX4MiFyFCGHFPm3VXVs146JFofPuUfUQ", + authorization_endpoint: "https://security.cloud.private/realms/nomadder/protocol/openid-connect/auth", + token_endpoint: "https://security.cloud.private/realms/nomadder/protocol/openid-connect/token", + userinfo_endpoint: "https://security.cloud.private/realms/nomadder/protocol/openid-connect/userinfo", + redirect_uri: "https://gitlab.cloud.private/users/auth/openid_connect/callback", + } + } +} +] + +``` + +https://gitlab.cloud.private/admin/application_settings/general + +* Edit Admin Area -> Setting -> General -> Sign-up restrictions -> Sign-out page URL and enter + https://security.cloud.private/realms/nomadder/protocol/openid-connect/logout?client_id=gitlab&post_logout_redirect_uri=https://gitlab.cloud.private + +Login to gitlab conatiner and execute + +``` shell +gitlab-ctl reconfigure +``` + +## Jenkins Help links + +* Current jenkins settings + https://jenkins.cloud.private/configuration-as-code/ +* List jenkins env vars + https://jenkins.cloud.private/env-vars.html/ + https://www.lambdatest.com/blog/set-jenkins-pipeline-environment-variables-list/ +* List plugin names + https://jenkins.cloud.private/pluginManager/api/json?tree=plugins[shortName] + https://jenkins.cloud.private/pluginManager/api/xml?tree=plugins[shortName] +* List all plugins + https://jenkins.cloud.private/pluginManager/api/json?depth=1 +* https://jenkins.cloud.private/pluginManager/api/xml?depth=1 +* Example jenkins file docker engine + https://github.com/moby/moby/blob/master/Jenkinsfile +* Nomad gitlab ci cd example +https://gitlab.com/internetarchive/nomad/-/blob/master/project.nomad#L118 +* Dynamic job names nomad + https://github.com/hashicorp/nomad/issues/9522 +* Github Darin Pope + https://github.com/darinpope +## Jenkins Gitlab integration + +### Prepare Gitlab + +1. Login to gitlab +2. Create an account jenkinsbot +3. Add this user to the groups you want tp build with jenkins with Maintainer access +4. Logout with admin and login with jenkinsbot +5. Create a api access token for jenkinsbot +6. Create a ssh key with ssh-keygen -t rsa -b 4096 -f id_rsa_gitlab -C "jenkinsbot@jenkins.cloud.private" +7. Import this ssh pub key for jenkinsbot user + +You can define group project or personal access token for acessing gitlab from jenkins +
It is more suitable to define a user named `jenkinsbot` and create a presonal access token for that user. +
after that you can give access to to this user for group and/or projects on gitlab + +## Prepare jenkins + +1. Login to jenkins +2. Create credentials got gitlab server: + 1. Gitlab Personal Access Token named `jenkinsbotAccessToken` Token ( with jenkinsbot gitlab access token ) + 2. Gitlab Api Token named `jenkinsbotApiToken` Token ( with jenkinsbot gitlab access token ) + 3. Secret text named `jenkinsbotWebhookToken` ( with jenkinsbot gitlab access token ) + 4. Secret ssh key named `jenkinsbotSshKey` ( with jenkinsbots' private key ) + 4. Or Secret username password named `jenkinsbotUserNamePassword` ( with jenkinsbots' private key ) + +3. Configure gitlab server with the + credentials `jenkinsbotAccessToken` , `jenkinsbotWebhookToken , jenkinsbotSshKey | jenkinsbotUserNamePassword` + +4. allow fetch local resources ( For import projects from local git instances ). I found Admin -> Settings -> Network -> Outbound Requests -> Allow requests to the local network from hooks and services +## Add git as remote in intelij + +1. git config http.sslVerify false + or for all repos git config --global http.sslVerify false + +## Post configuration Gitlab Webhooks + +The Webhook plugin nedded for trieggering multibranc job types in jenkins +The jenkins integration in gitlab works only with normal pipeline jobs + +1. In projects settings goto webhook and type the uri for webhook. + http(s):///multibranch-webhook-trigger/invoke?token= + e.g. https://jenkins.cloud.private/multibranch-webhook-trigger/invoke?token=11659c4ca9e551b27f86c46e750c5e7440 +3. Do not forget to santinize the token un the url +3. Enable auto merge + in [gitlab](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html) + +Gitlab jenkins Token: glpat-TWYXxAsrQCPk4MqNiJGy +Jenkins api token: 1167f9e99cd8d64fa9ae532a46ce1e5d0f + +
+ [How to gen SSH Key](https://geoweb.princeton.edu/research/jenkins-doc/tutorial_gitlab.html) +
+ [Jenkins webook multibranch](https://santoshk.dev/posts/2022/how-to-setup-a-github-to-jenkins-pipeline-with-webhook/) +## TODOS + +1. HOW to manage users over keycloak so that we don't need a user management in gitlab +2. Import cluster certificate to jenkins \ No newline at end of file diff --git a/test_deployments/cicd_deployment/TODO_Jenkins.MD b/test_deployments/cicd_deployment/TODO_Jenkins.MD new file mode 100644 index 0000000..616356d --- /dev/null +++ b/test_deployments/cicd_deployment/TODO_Jenkins.MD @@ -0,0 +1,13 @@ +* Trigger build on push +* Filter for Trigger on push +* Trigger build on Merge request +* Trigger Nightly build +* Clean up workspace +* settings.xml per project +* Delete .m2 before build ? +* .m2 config options + * global on build node + * defined in settgins.xml + * local to the executor + * local to the workspace + \ No newline at end of file diff --git a/test_deployments/cicd_deployment/certgen_test.nomad b/test_deployments/cicd_deployment/certgen_test.nomad new file mode 100644 index 0000000..306b9bc --- /dev/null +++ b/test_deployments/cicd_deployment/certgen_test.nomad @@ -0,0 +1,53 @@ +job "certgentest"{ + type= "batch" + group "certgentest"{ + count = 1 + volume "ca_cert" { + type = "host" + source = "ca_cert" + read_only = true + } + task "03-jenknins-cert-import" { + + volume_mount { + volume = "ca_cert" + destination = "/certs" + } + + driver = "docker" + + config { + image = "jenkins/jenkins:2.387.3-lts-jdk17" + command = "/bin/sh" +# args = ["-c", "/var/jenkins_home/gen.sh"] + args = ["-c", "sleep 3000"] + volumes = ["local/gen.sh:/var/jenkins_home/gen.sh"] + } + + template { + perms = "777" + data = < /dev/null | openssl x509 -outform PEM > /var/jenkins_home/root_ca.pem || EXIT_STATUS=$? +#${JAVA_HOME}/bin/keytool -import -trustcacerts -alias gitlabca -keystore /var/jenkins_home/cacerts -file /var/jenkins_home/root_ca.pem -noprompt -storepass changeit || EXIT_STATUS=$? +mkdir -p ${NOMAD_ALLOC_DIR}/data/security +cp /var/jenkins_home/cacerts ${NOMAD_ALLOC_DIR}/data/security || EXIT_STATUS=$? +echo "Finished gen cert" || EXIT_STATUS=$? +exit $EXIT_STATUS +EOF + destination = "local/gen.sh" + change_mode = "noop" + } + resources { + cpu = 200 + memory = 128 + } + } + } +} \ No newline at end of file diff --git a/test_deployments/cicd_deployment/cicd.volumes b/test_deployments/cicd_deployment/cicd.volumes new file mode 100644 index 0000000..391c2ee --- /dev/null +++ b/test_deployments/cicd_deployment/cicd.volumes @@ -0,0 +1,19 @@ +sudo mkdir -p /opt/nomadvolumes/cicd/gitlab/etc +sudo mkdir -p /opt/nomadvolumes/cicd/gitlab/opt +sudo mkdir -p /opt/nomadvolumes/cicd/jenkins + + +client { + host_volume "nomad_volume_stack_cicd_gitlab_etc" { + path = "/opt/nomadvolumes/cicd/gitlab/etc" + read_only = false + } + host_volume "nomad_volume_stack_cicd_gitlab_opt" { + path = "/opt/nomadvolumes/cicd/gitlab/opt" + read_only = false + } + host_volume "nomad_volume_stack_cicd_jenkins" { + path = "/opt/nomadvolumes/cicd/jenkins" + read_only = false + } +} \ No newline at end of file diff --git a/test_deployments/cicd_deployment/cicd_with_volumes.nomad b/test_deployments/cicd_deployment/cicd_with_volumes.nomad new file mode 100644 index 0000000..e61f06b --- /dev/null +++ b/test_deployments/cicd_deployment/cicd_with_volumes.nomad @@ -0,0 +1,664 @@ + +variable "registry" { + type = string + description = "The private docker registry" + default = "registry.cloud.private" +} + +variable "image_jenkins" { + type = string + description = "The used jenkins image" + default = "jenkins/jenkins:2.401.1-lts-jdk17_1" +} + +variable "image_gitlab" { + type = string + description = "The used jenkins image" + default = "gitlab/gitlab-ce:16.0.3-ce.0" +} + +# See https://github.com/hashicorp/nomad-pack-community-registry/blob/main/packs/jenkins/templates/jenkins.nomad.tpl +job "cicd-job" { + datacenters = ["nomadder1"] + type = "service" + +# Place the whole job on the same node +# This can be moved to group level as well + constraint { + attribute = "${attr.unique.hostname}" + set_contains = "worker-02" + } + + reschedule { + delay = "10s" + delay_function = "constant" + unlimited = true + } + + update { + health_check = "checks" + max_parallel = 1 + # Alloc is marked as unhealthy after this time + healthy_deadline = "5m" + auto_revert = true + # Mark the task as healthy after 10s positive check + min_healthy_time = "10s" + # Task is dead after failed checks in 1h + progress_deadline = "1h" + } + + group gitlab-group{ + count =1 + volume "nomad_volume_stack_cicd_gitlab_etc" { + type = "host" + source = "nomad_volume_stack_cicd_gitlab_etc" + read_only = false + } + volume "nomad_volume_stack_cicd_gitlab_opt" { + type = "host" + source = "nomad_volume_stack_cicd_gitlab_opt" + read_only = false + } + + network { + mode = "bridge" + port "http" { + to = 80 + } + port "http2" { + to = 80 + } + port "http3" { + to = 80 + } + port "ssl" { + to = 22 + } + } + + service { + name = "gitlab-service" + port = "http" + tags = [ + "traefik.enable=true", + "traefik.consulcatalog.connect=false", + "traefik.http.routers.gitlab-service.tls=true", + "traefik.http.routers.gitlab-service.rule=Host(`gitlab.cloud.private`)", + ] + check { + name = "readiness" + type = "http" + path = "/-/readiness" + interval = "10s" + timeout = "2s" + check_restart { + limit = 3 + grace = "300s" + ignore_warnings = false + } + } + } + + service { + name = "gitlab-liveness" + port = "http2" + check { + name = "readiness" + type = "http" + path = "/-/liveness" + interval = "10s" + timeout = "2s" + } + } + service { + name = "gitlab-health" + port = "http3" + check { + name = "readiness" + type = "http" + path = "/-/health" + interval = "10s" + timeout = "2s" + } + } + task "gitlab-task" { + volume_mount { + volume = "nomad_volume_stack_cicd_gitlab_etc" + destination = "/etc/gitlab" + } + volume_mount { + volume = "nomad_volume_stack_cicd_gitlab_opt" + destination = "/var/opt/gitlab" + } + + + driver = "docker" + + config { + image = "registry.cloud.private/${var.image_gitlab}" + ports = ["http","ssl"] + image_pull_timeout = "10m" +# volumes = [] + } + resources { + cpu = 2000 + memory = 8092 + } + env { + GITLAB_ROOT_EMAIL="root@local" + GITLAB_ROOT_PASSWORD="lcl@admin" + #GITLAB_OMNIBUS_CONFIG = "external_url 'https://gitlab.cloud.private'; gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0']" + GITLAB_OMNIBUS_CONFIG = "external_url 'https://gitlab.cloud.private'; nginx['listen_https'] = false; nginx['listen_port'] = 80; gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0']" + } + } + } + + group "jenkins-group" { + volume "nomad_volume_stack_cicd_jenkins" { + type = "host" + source = "nomad_volume_stack_cicd_jenkins" + read_only = false + } + volume "ca_cert" { + type = "host" + source = "ca_cert" + read_only = true + } + volume "cert_docker" { + type = "host" + source = "cert_docker" + read_only = true + } + + volume "cert_nomad" { + type = "host" + source = "cert_nomad" + read_only = true + } + + volume "cert_consul" { + type = "host" + source = "cert_consul" + read_only = true + } + + count = 1 + + network { + mode = "bridge" + port "http" { + to = 8080 + } + port "http2" { + to = 8080 + } + port "jnlp" { + to = 50000 + } + } + + service { + name = "jenkins-service" + port = "http" + tags = [ + "traefik.enable=true", + "traefik.consulcatalog.connect=false", + "traefik.http.routers.jenkins-service.tls=true", + "traefik.http.routers.jenkins-service.rule=Host(`jenkins.cloud.private`)", + ] + check { + name = "alive" + type = "http" + path = "/login" + interval = "10s" + timeout = "2s" + check_restart { + limit = 3 + grace = "300s" + ignore_warnings = false + } + } + + } + service { + name = "jenkins-resources-service" + port = "http2" + tags = [ + "traefik.enable=true", + "traefik.consulcatalog.connect=false", + "traefik.http.routers.jenkins-resources-service.tls=true", + "traefik.http.routers.jenkins-resources-service.rule=Host(`jenkins-resources.cloud.private`)", + ] + check { + name = "alive" + type = "http" + path = "/login" + interval = "10s" + timeout = "2s" + check_restart { + limit = 3 + grace = "300s" + ignore_warnings = false + } + } + } +# Enable this if volume is enabled + task "01-jenknins-chown" { + lifecycle { + hook = "prestart" + sidecar = false + } + + volume_mount { + volume = "nomad_volume_stack_cicd_jenkins" + destination = "/var/jenkins_home" + } + + driver = "docker" + + config { + image = "registry.cloud.private/busybox:stable" + command = "sh" + args = ["-c", "chown -R 1000:1000 /var/jenkins_home"] + } + + resources { + cpu = 200 + memory = 128 + } + } + + task "02-jenknins-plugins" { + driver = "docker" + volume_mount { + volume = "nomad_volume_stack_cicd_jenkins" + destination = "/var/jenkins_home" + } + config { + image = "${var.registry}/${var.image_jenkins}" + command = "jenkins-plugin-cli" + args = ["--verbose","-f", "/var/jenkins_home/plugins.txt", "--plugin-download-directory", "/var/jenkins_home/plugins/"] + volumes = ["local/plugins.txt:/var/jenkins_home/plugins.txt"] + } + + lifecycle { + hook = "prestart" + sidecar = false + } +# List of Plugins https://updates.jenkins.io/download/plugins/ +# For config the plugin cli see https://github.com/jenkinsci/plugin-installation-manager-tool + +# OIDC Login with keycloak and jenkins https://github.com/jenkinsci/keycloak-plugin + template { + data = < /dev/null | openssl x509 -outform PEM > /var/jenkins_home/root_ca.pem || EXIT_STATUS=$? +#${JAVA_HOME}/bin/keytool -import -trustcacerts -alias gitlabca -keystore /var/jenkins_home/cacerts -file /var/jenkins_home/root_ca.pem -noprompt -storepass changeit || EXIT_STATUS=$? +mkdir -p ${NOMAD_ALLOC_DIR}/data/security +cp /var/jenkins_home/cacerts ${NOMAD_ALLOC_DIR}/data/security || EXIT_STATUS=$? +echo "Finished gen cert" || EXIT_STATUS=$? +mkdir -p ${NOMAD_ALLOC_DIR}/data/docker_certs +cp /certs/cluster-ca-bundle.pem ${NOMAD_ALLOC_DIR}/data/docker_certs/ca.pem || EXIT_STATUS=$? +cp /certsdocker/docker-client.pem ${NOMAD_ALLOC_DIR}/data/docker_certs/cert.pem || EXIT_STATUS=$? +cp /certsdocker/docker-client-key.pem ${NOMAD_ALLOC_DIR}/data/docker_certs/key.pem || EXIT_STATUS=$? +exit $EXIT_STATUS +EOF + destination = "local/gen.sh" + change_mode = "noop" + } + resources { + cpu = 200 + memory = 128 + } + } + + task "jenkins-task" { + driver = "docker" + volume_mount { + volume = "nomad_volume_stack_cicd_jenkins" + destination = "/var/jenkins_home" + } + + // Need for os level action like git + volume_mount { + # OS certifate folders + # "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. + #"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 + #"/etc/ssl/ca-bundle.pem", // OpenSUSE + #"/etc/pki/tls/cacert.pem", // OpenELEC + #"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 + #"/etc/ssl/cert.pem", // Alpine Linux + volume = "ca_cert" + destination = "/etc/ssl/certs/" + } + + volume_mount { + volume = "cert_consul" + destination = "/etc/opt/certs/consul" + } + + volume_mount { + volume = "cert_nomad" + destination = "/etc/opt/certs/nomad" + } + + config { + image = "${var.registry}/${var.image_jenkins}" + ports = ["http","jnlp"] + volumes = [ + "local/jasc.yaml:/var/jenkins_home/jenkins.yaml", +# "../${NOMAD_ALLOC_DIR}/data/security:/etc/ssl/certs/java/cacerts", + "../${NOMAD_ALLOC_DIR}/data/security/cacerts:/opt/java/openjdk/lib/security/cacerts", + "../${NOMAD_ALLOC_DIR}/data/docker_certs/:/var/jenkins_home/.docker", + "local/settings.xml:/var/jenkins_home/.m2/settings.xml", + ] + } + env{ + JAVA_OPTS ="-Djava.awt.headless=true -Djenkins.install.runSetupWizard=false -Dhudson.model.DownloadService.noSignatureCheck=true" + DOCKER_HOST = "10.21.21.41" + DOCKER_TLS_VERIFY = 1 + DOCKER_BUILDKIT = 1 + //Nomad address with port + NOMAD_ADDR="https://${attr.nomad.advertise.address}" + NOMAD_CACERT="/etc/ssl/certs/cluster-ca-bundle.pem" + NOMAD_CLIENT_CERT="/etc/opt/certs/nomad/nomad-cli.pem" + NOMAD_CLIENT_KEY="/etc/opt/certs/nomad/nomad-cli-key.pem" + CONSUL_CACERT="/etc/ssl/certs/cluster-ca-bundle.pem" + CONSUL_HTTP_SSL=true + CONSUL_HTTP_SSL_VERIFY=true + CONSUL_HTTP_ADDR="${attr.unique.network.ip-address}:8501" + CONSUL_CLIENT_KEY="/etc/opt/certs/consul/consul-key.pem" + CONSUL_CLIENT_CERT="/etc/opt/certs/consul/consul.pem" + } + template { + right_delimiter = "++" + left_delimiter = "++" + change_mode = "noop" + destination = "local/settings.xml" + data = < + + + + + releases + development + development123 + + + snapshots + development + development123 + + + + + + nexus + + + + + nexusLocal + + https://nexus.cloud.private/repository/maven-public/ + + true + never + + + true + always + + + + + + nexusLocalPlugins + + https://nexus.cloud.private/repository/maven-public/ + + true + + + true + always + + + + + + + + + nexus + + + + + + + +EOF + } +# configuration-as-code plugin is required for that +# https://plugins.jenkins.io/configuration-as-code/ +# Export current settings with https://jenkins.cloud.private/configuration-as-code/ + + template { + right_delimiter = "++" + left_delimiter = "++" + data = < + +USER root +RUN apt -y update && apt -y install jq +USER jenkins \ No newline at end of file diff --git a/test_deployments/cicd_deployment/jenkinsbuild/build.sh b/test_deployments/cicd_deployment/jenkinsbuild/build.sh new file mode 100644 index 0000000..e786819 --- /dev/null +++ b/test_deployments/cicd_deployment/jenkinsbuild/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash +export PULL_IMAGE=jenkins/jenkins:2.401.1-lts-jdk17 +export PUSH_IMAGE=jenkins/jenkins:2.401.1-lts-jdk17_1 + + if [ -z "$PULL_REGISTRY" ]; then + echo "PULL_REGISTRY is not specified" + exit 1 + fi + + if [ -z "$PUSH_REGISTRY" ]; then + echo "PUSH_REGISTRY is not specified" + exit 1 + fi + + echo "Build version $APP_VERSION of $APP_NAME and push to $PUSH_REGISTRY" + docker build --build-arg PULL_REGISTRY=$PULL_REGISTRY --build-arg IMAGE=$PULL_IMAGE -t $PUSH_REGISTRY/$PUSH_IMAGE . + echo docker push $PUSH_REGISTRY/$PUSH_IMAGE + docker push $PUSH_REGISTRY/$PUSH_IMAGE + diff --git a/test_deployments/elasticsearch.nomad b/test_deployments/elasticsearch.nomad index 2f2c9a1..813ad35 100644 --- a/test_deployments/elasticsearch.nomad +++ b/test_deployments/elasticsearch.nomad @@ -69,7 +69,9 @@ job "es-cluster" { "-Etransport.tcp.port=${NOMAD_PORT_transport}", # Transport port (originally port 9300) inside the container "-Expack.license.self_generated.type=basic", # use x-packs basic license (free) ] - +# sysctl = { +# "net.core.somaxconn" = "16384" +# } ulimit { # ensure elastic search can lock all memory for the JVM on start memlock = "-1" diff --git a/test_deployments/setup.sh b/test_deployments/setup.sh deleted file mode 100644 index 0994383..0000000 --- a/test_deployments/setup.sh +++ /dev/null @@ -1,406 +0,0 @@ -dnf update -y -dnf install -y curl jq unzip - -export CLUSTER_DC=fra1 -export CLUSTER_PRIMARY_DC=fra1 -export CLUSTER_PRIVATE_IPS=1.1.2.2,2.2.3.3,3.3.4.4 -export CLUSTER_PUBLIC_IPS=1.1.1.1,2.2.2.2,3.3.3.3 -export CLUSTER_SIZE=$(echo $CLUSTER_PUBLIC_IPS | jq -Rc 'split(",") | length') - -export DOCKER_USERNAME="" -export DOCKER_PASSWORD="" - -export HOST_NAME=$(hostname) -export HOST_PUBLIC_IP=$(hostname -I | cut -d " " -f 1) -export HOST_PRIVATE_IP=$(hostname -I | cut -d " " -f 3) - - -## -# NODES -## - -# DOCKER - -# Install Docker -dnf config-manager \ - --add-repo https://download.docker.com/linux/fedora/docker-ce.repo - -dnf config-manager \ - --set-enabled docker-ce-edge - -dnf config-manager \ - --set-enabled docker-ce-test - -dnf install -y docker-ce - -# Start Docker -systemctl enable docker -systemctl start docker - -# Generate default credentials -echo "$DOCKER_PASSWORD" | docker login \ - --username "$DOCKER_USERNAME" \ - --password-stdin - -mv $HOME/.docker/config.json /etc/nomad.d/docker-config.json -chown nomad:nomad /etc/nomad.d/docker-config.json - - -# DNSMASQ - -# Install DNSMasq -dnf install -y dnsmasq - -cat << EOF > /etc/dnsmasq.d/10-consul.conf -no-poll -no-resolv -rev-server=0.0.0.0/8,127.0.0.1#8600 -rev-server=10.0.0.0/8,127.0.0.1#8600 -rev-server=127.0.0.1/8,127.0.0.1#8600 -rev-server=169.254.0.0/16,127.0.0.1#8600 -rev-server=192.168.0.0/16,127.0.0.1#8600 -server=/consul/127.0.0.1#8600 -server=67.207.67.2 -server=67.207.67.3 -server=8.8.8.8 -server=8.8.4.4 -EOF - -cat << EOF > /etc/resolv.conf -nameserver 127.0.0.1 -search localdomain -EOF - -systemctl enable dnsmasq -systemctl start dnsmasq - - -# CONSUL -# https://www.consul.io/docs -# https://learn.hashicorp.com/consul/advanced/day-1-operations/deployment-guide -export CONSUL_BOOTSTRAP_TOKEN= -export CONSUL_ENCRYPTION_KEY= - -# Create user and group -groupadd -r consul -useradd -Mr -g consul -s /usr/sbin/nologin consul - -# Intall Consul -curl -LSs https://releases.hashicorp.com/consul/1.4.3/consul_1.4.3_linux_amd64.zip -o /tmp/consul.zip -unzip -oq -d /usr/local/bin /tmp/consul.zip -rm -f /tmp/consul.zip -chmod 755 /usr/local/bin/consul -chown root:root /usr/local/bin/consul -mkdir -p /etc/consul.d -mkdir -p /var/tmp/consul -chown consul:consul /var/tmp/consul - -# Install autocomplete -consul -autocomplete-install -complete -C /usr/local/bin/consul consul - -# Create configuration -export CONSUL_ENCRYPTION_KEY=$(consul keygen) - -cat << EOF > /etc/consul.d/config.json -{ - "advertise_addr": "${HOST_PRIVATE_IP}", - "advertise_addr_wan": "${HOST_PUBLIC_IP}", - "bootstrap_expect": ${CLUSTER_SIZE}, - "datacenter": "${CLUSTER_DC}", - "primary_datacenter": "${CLUSTER_PRIMARY_DC}", - "data_dir": "/var/tmp/consul", - "encrypt": "${CONSUL_ENCRYPTION_KEY}", - "server": true, - "ui": true, - "acl": { - "enabled": true, - "default_policy": "allow" - }, - "performance": { - "raft_multiplier": 1 - }, - "retry_join": $(echo $CLUSTER_PRIVATE_IPS | jq -Rc 'split(",")') -} -EOF - -# Create service -cat << EOF > /lib/systemd/system/consul.service -[Unit] -Description=Consul -Requires=network-online.target -After=network-online.target -ConditionDirectoryNotEmpty=/etc/consul.d -[Service] -User=consul -Group=consul -RuntimeDirectory=consul -PIDFile=/var/run/consul.pid -PermissionsStartOnly=true -ExecStart=/usr/local/bin/consul agent -config-dir /etc/consul.d -pid-file /var/run/consul/consul.pid -ExecReload=/bin/kill -HUP $MAINPID -KillMode=process -KillSignal=SIGTERM -Restart=on-failure -RestartSec=15s -[Install] -WantedBy=multi-user.target -EOF - -# Start service -systemctl enable consul -systemctl start consul - -# Create ACL bootstrap token. -# Run once all Consul agents are running and a leader is elected -# https://learn.hashicorp.com/consul/advanced/day-1-operations/acl-guide#step-2-create-the-bootstrap-token -# https://learn.hashicorp.com/consul/advanced/day-1-operations/acl-guide#step-3-create-an-agent-token-policy -consul acl bootstrap \ - -datacenter ${CLUSTER_PRIMARY_DC} - -export CONSUL_BOOTSTRAP_TOKEN="" - -consul acl set-agent-token master ${CONSUL_BOOTSTRAP_TOKEN} - - -# VAULT -# https://www.vaultproject.io/docs -# https://learn.hashicorp.com/vault/day-one/ops-vault-ha-consul -export VAULT_ADDR=http://127.0.0.1:8200 -export VAULT_TOKEN="" - -# Create user and group -groupadd -r vault -useradd -Mr -g vault -s /usr/sbin/nologin vault - -# Install Vault -curl -LSs https://releases.hashicorp.com/vault/1.0.3/vault_1.0.3_linux_amd64.zip -o /tmp/vault.zip -unzip -oq -d /usr/local/bin /tmp/vault.zip -rm -f /tmp/vault.zip -chmod 755 /usr/local/bin/vault -chown root:root /usr/local/bin/vault -mkdir -p /etc/vault.d - -# Create configuration -cat << EOF > /etc/vault.d/config.json -{ - "api_addr": "http://${HOST_PRIVATE_IP}:8200", - "cluster_addr": "http://${HOST_PRIVATE_IP}:8201", - "pid_file": "", - "ui": true, - "listener": { - "tcp": { - "address": "0.0.0.0:8200", - "tls_disable": true - } - }, - "storage": { - "consul": { - "address": "127.0.0.1:8500", - "path": "vault/" - } - } -} -EOF - -# Create service -cat << EOF > /lib/systemd/system/vault.service -[Unit] -Description=Vault -Requires=network-online.target -After=network-online.target -ConditionFileNotEmpty=/etc/vault.d/config.json -[Service] -User=vault -Group=vault -RuntimeDirectory=vault -PIDFile=/var/run/vault/vault.pid -PermissionsStartOnly=true -ExecStart=/usr/local/bin/vault server -config /etc/vault.d/config.json -ExecReload=/bin/kill -HUP $MAINPID -KillMode=process -KillSignal=SIGTERM -Restart=on-failure -RestartSec=15s -LimitNOFILE=65536 -LimitMEMLOCK=infinity -[Install] -WantedBy=multi-user.target -EOF - -# Grant access to memory lock on host -setcap cap_ipc_lock=+ep $(readlink -f $(which vault)) - -# Start service -systemctl enable vault -systemctl start vault - -# Initialise and unseal. -# Run once all Vault agents are running -vault operator init - -export VAULT_TOKEN="" - -vault operator unseal - -# Upgrade default key/value engine to v2 -vault kv enable-versioning secret/ - -# Enable Consul engine -vault secrets enable consul - -# Enable Nomad engine. -# Only once Nomad cluster is ready -vault secrets enable nomad - - -# NOMAD -# https://www.nomadproject.io/docs - -# Create user and group -groupadd -r nomad -useradd -Mr -G docker -g nomad -s /usr/sbin/nologin nomad - -# Install Nomad -curl -LSs https://releases.hashicorp.com/nomad/0.8.7/nomad_0.8.7_linux_amd64.zip -o /tmp/nomad.zip -unzip -oq -d /usr/local/bin /tmp/nomad.zip -rm -f /tmp/nomad.zip -chown root:root /usr/local/bin/nomad -chmod 755 /usr/local/bin/nomad -mkdir -p /etc/nomad.d -mkdir -p /var/tmp/nomad -chown nomad:nomad /var/tmp/nomad - -# Install autocomplete -nomad -autocomplete-install -complete -C /usr/local/bin/nomad nomad - -# Create configuration -cat << EOF > /etc/nomad.d/config.json -{ - "datacenter": "${CLUSTER_DC}", - "data_dir": "/var/tmp/nomad", - "advertise": { - "http": "${HOST_PRIVATE_IP}", - "rpc": "${HOST_PRIVATE_IP}", - "serf": "${HOST_PRIVATE_IP}" - }, - "client": { - "enabled": true, - "node_class": "standard", - "options": { - "docker.auth.config": "/etc/nomad.d/docker-config.json" - } - }, - "server": { - "bootstrap_expect": ${CLUSTER_SIZE}, - "enabled": true - }, - "consul": { - "address": "127.0.0.1:8500", - "auto_advertise": true, - "client_auto_join": true - }, - "vault": { - "address": "http://vault.service.consul:8200", - "enabled": true, - "token": "${VAULT_TOKEN}" - } -} -EOF - -# Create service -cat << EOF > /lib/systemd/system/nomad.service -[Unit] -Description=Nomad -Requires=network-online.target -After=network-online.target -ConditionFileNotEmpty=/etc/nomad.d/config.json -[Service] -User=nomad -Group=nomad -RuntimeDirectory=nomad -PIDFile=/var/run/nomad/nomad.pid -PermissionsStartOnly=true -ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d/config.json -ExecReload=/bin/kill -HUP $MAINPID -KillMode=process -KillSignal=SIGTERM -Restart=on-failure -RestartSec=15s -[Install] -WantedBy=multi-user.target -EOF - -# Start service -systemctl enable nomad -systemctl start nomad - - -## -# LOAD BALANCER -## - -# CONSUL (CLIENT) - -# Create configuration -cat << EOF > /etc/consul.d/config.json -{ - "advertise_addr": "${HOST_PRIVATE_IP}", - "advertise_addr_wan": "${HOST_PUBLIC_IP}", - "client_addr": "127.0.0.1 ${HOST_PUBLIC_IP}", - "datacenter": "${CLUSTER_DC}", - "primary_datacenter": "${CLUSTER_PRIMARY_DC}", - "encrypt": "${CONSUL_ENCRYPTION_KEY}", - "server": false, - "ui": true, - "acl": { - "enabled": true, - "default_policy": "allow" - }, - "retry_join": $(echo $CLUSTER_PRIVATE_IPS | jq -Rc 'split(",")') -} -EOF - -# Create service -# ... - -# Start service -# ... - -# Create ACL bootstrap token -# ... - - -# NOMAD (CLIENT) - -# Create configuration -cat << EOF > /etc/nomad.d/config.json -{ - "datacenter": "${CLUSTER_DC}", - "data_dir": "/var/tmp/nomad", - "advertise": { - "http": "${HOST_PRIVATE_IP}", - "rpc": "${HOST_PRIVATE_IP}", - "serf": "${HOST_PRIVATE_IP}" - }, - "client": { - "enabled": true, - "network_adapter": "", - "node_class": "load-balancer" - }, - "server": { - "enabled": false - }, - "consul": { - "address": "127.0.0.1:8500", - "auto_advertise": true, - "client_auto_join": true - }, - "vault": { - "address": "http://vault.service.consul:8200", - "enabled": true, - "token": "${VAULT_TOKEN}" - } -} -EOF \ No newline at end of file diff --git a/test_deployments/shared_volume.nomad b/test_deployments/shared_volume.nomad new file mode 100644 index 0000000..a91800a --- /dev/null +++ b/test_deployments/shared_volume.nomad @@ -0,0 +1,44 @@ +job "sharedVolume" { + + group "grp" { + # ephemeral_disk { + # migrate = true + # size = 500 + # sticky = true + # } + task "a" { + driver = "docker" + + config { + image = "busybox:1.36.1" + command = "/bin/sh" + args = ["-c", "sleep 300"] + # volumes = ["../${NOMAD_ALLOC_DIR}/data:/shared/srv"] + mount { + type = "bind" + source = "../${NOMAD_ALLOC_DIR}/data" + target = "/shared/srv" + readonly = false + } + } + + } + + task "b" { + driver = "docker" + + config { + image = "busybox:1.36.1" + command = "/bin/sh" + args = ["-c", "sleep 300"] + # volumes = ["../${NOMAD_ALLOC_DIR}/data:/shared/srv"] + mount { + type = "bind" + source = "../${NOMAD_ALLOC_DIR}/data" + target = "/shared/srv" + readonly = false + } + } + } + } +} \ No newline at end of file