Merge branch 'main' into improve_readme

.github/workflows/main.yml

@@ -7,4 +7,4 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Run ansible-lint
-        uses: ansible-community/ansible-lint-action@main
+        uses: ansible-community/ansible-lint-action@v6.11.0

group_vars/all/50-o11y

@@ -17,14 +17,19 @@ o11y_enabled_roles:
 o11y_grafana_agent_dir: "/opt/grafana-agent"
 o11y_grafana_agent_wal_dir: "/var/lib/grafana-agent/wal"
 o11y_grafana_agent_pos_dir: "/var/lib/grafana-agent/positions"
+o11y_grafana_agent_unit: "/etc/systemd/system/telemetry.service"
 
 o11y_grafana_agent_dirs:
   - "{{ o11y_grafana_agent_dir }}"
   - "{{ o11y_grafana_agent_wal_dir }}"
   - "{{ o11y_grafana_agent_pos_dir }}"
 
-grafana_agent_pkg: https://github.com/grafana/agent/releases/download/v0.22.0/agent-linux-amd64.zip
+grafana_agent_pkg: "https://github.com/grafana/agent/releases/download/v0.32.1/grafana-agent-linux-{{ ubuntu_arch }}.zip"
 
 o11y_postgres_exporter_dir: "/opt/postgres-exporter"
-
 pg_exp_pkg: "https://github.com/prometheus-community/postgres_exporter/releases/download/v0.11.1/postgres_exporter-0.11.1.linux-{{ ubuntu_arch }}.tar.gz"
+pg_exp_conn: "DATA_SOURCE_NAME='postgresql://{{ maas_postgres_user }}:{{ maas_postgres_password }}@\
+              {{ inventory_hostname }}:5432/{{ maas_postgres_database }}?sslmode=disable'"
+
+ha_exp_pkg: "https://github.com/ClusterLabs/ha_cluster_exporter/releases/download/1.3.1/ha_cluster_exporter-{{ ubuntu_arch }}.gz"
+ha_exp_dir: "/opt/ha_cluster_exporter"

group_vars/maas_pacemaker/01-pacemaker

@@ -4,3 +4,9 @@ maas_paf_deb_url: "https://github.com/ClusterLabs/PAF/releases/download/v{{ maas
 maas_paf_deb_dest: "/tmp/resource-agents-paf_{{maas_paf_version }}-{{ maas_paf_revision }}_all.deb"
 
 maas_pacemaker_self_address: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] if 'ansible_default_ipv4' in hostvars[inventory_hostname] else hostvars[inventory_hostname]['ansible_default_ipv6']['address'] }}"
+
+maas_pacemaker_noproxy_list_v4: "{{ groups['maas_pacemaker'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | list }}"
+
+maas_pacemaker_noproxy_list_v6: "{{ groups['maas_pacemaker'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | list }}"
+
+maas_pacemaker_noproxy_list: "{% if maas_pacemaker_noproxy_list_v4 %}{{ maas_pacemaker_noproxy_list_v4 | join(',') }}{% endif %}{% if maas_pacemaker_noproxy_list_v4 and maas_pacemaker_noproxy_list_v6 %},{% endif %}{% if maas_pacemaker_noproxy_list_v6 %}{{ maas_pacemaker_noproxy_list_v6 }}{% endif %}"

group_vars/maas_postgres/60-firewall

@@ -1,6 +1,6 @@
 ---
-maas_open_tcp_ports: 
+maas_pg_tcp_ports: 
 - 5432
 - "{{ 9187 if o11y_enable else (false) }}"
 
-maas_open_udp_ports: []
+maas_pg_udp_ports: []

group_vars/maas_postgres_proxy/60-firewall

@@ -1,6 +1,6 @@
 ---
-maas_open_tcp_ports:
+maas_pgproxy_tcp_ports:
 - 5432
 - "{{ 5051 if 'maas_postgres' in group_names else (false) }}"
 
-maas_open_udp_ports:
+maas_pgproxy_udp_ports:

group_vars/maas_proxy/60-firewall

@@ -1,6 +1,6 @@
 ---
-maas_open_tcp_ports:
+maas_proxy_tcp_ports:
 - "{{ 5432 if 'maas_postgres' in group_names else (false) }}" 
 - "{{ maas_proxy_port }}"
 
-maas_open_udp_ports:
+maas_proxy_udp_ports:

group_vars/maas_rack_controller/60-firewall

@@ -1,13 +1,13 @@
 ---
-maas_open_tcp_ports:
+maas_rack_tcp_ports:
 - 53		# dns
 - 514 		# rsyslog
 - 5248		# rack http port
-- 3218
+- 3128
 - 8000
 - "{{ maas_promtail_port if o11y_enable else (false) }}"
 
-maas_open_udp_ports:
+maas_rack_udp_ports:
 - 53 		# dns
 - 67:69 	# tftp, dhcp 
 - 123		# ntp

group_vars/maas_region_controller/60-firewall

@@ -1,5 +1,5 @@
 ---
-maas_open_tcp_ports: 
+maas_rack_tcp_ports: 
 - 53 			# dns
 - 514 			# rsyslog
 - 3128
@@ -13,7 +13,7 @@ maas_open_tcp_ports:
 - "{{ maas_proxy_postgres_port if 'maas_proxy' in group_names else (false) }}"
 - "{{ maas_promtail_port if o11y_enable else (false) }}"
 
-maas_open_udp_ports:
+maas_region_udp_ports:
 - 53 			# dns
 - 123 			# ntp
 - 514 			# rsyslog

roles/maas_firewall/tasks/setup_firewall_rules.yaml

@@ -53,17 +53,25 @@
     protocol: tcp
     destination_port: "{{ item }}"
     jump: ACCEPT
-  with_items: '{{ maas_open_tcp_ports | select() }}'
-  when: maas_open_tcp_ports
+  with_items:
+    - '{{ maas_pg_tcp_ports | default([]) | select() }}'
+    - '{{ maas_pgproxy_tcp_ports | default([]) | select() }}'
+    - '{{ maas_proxy_tcp_ports | default([]) | select() }}'
+    - "{{ maas_rack_tcp_ports | default([]) | select() }}"
+    - "{{ maas_region_tcp_ports | default([]) | select() }}"
 
 - name: Open udp ports
   ansible.builtin.iptables:
     chain: INPUT
     protocol: udp
     destination_port: "{{ item }}"
     jump: ACCEPT
-  with_items: "{{ maas_open_udp_ports }}"
-  when: maas_open_udp_ports
+  with_items:
+    - "{{ maas_pg_udp_ports | default([]) | select() }}"
+    - "{{ maas_pgproxy_udp_ports | default([]) | select() }}"
+    - "{{ maas_proxy_udp_ports | default([]) | select() }}"
+    - "{{ maas_rack_udp_ports | default([]) | select() }}"
+    - "{{ maas_region_udp_ports | default([]) | select() }}"
 
 - name: Set policy for INPUT chain to drop (otherwise)
   ansible.builtin.iptables:

roles/maas_pacemaker/handlers/main.yaml

@@ -11,3 +11,10 @@
     name: "sshd.service"
     state: restarted
     no_block: false
+
+- name: "Start HA clusters metrics agent"
+  ansible.builtin.systemd:
+    name: ha_cluster_exporter
+    enabled: true
+    state: "restarted"
+    daemon-reload: true

roles/maas_pacemaker/tasks/main.yaml

@@ -20,6 +20,15 @@
     group: "root"
     mode: 0644
 
+- name: "Setup NOPROXY for cluster members"
+  ansible.builtin.lineinfile:
+    dest: "{{ item }}"
+    line: "NOPROXY={{ groups['maas_pacemaker'] | join(',') }},{{ maas_pacemaker_noproxy_list }},{{ maas_postgres_floating_ip }}"
+  with_items:
+    - "/etc/default/pacemaker"
+    - "/etc/default/pcsd"
+  when: proxy_env
+
 - name: "Generate Pacemaker user Password"
   ansible.builtin.command: openssl rand -base64 14
   register: maas_pacemaker_user_password_output
@@ -65,6 +74,9 @@
   until: pacemaker_auth is not failed
   retries: 3
   delay: 2
+  environment:
+    no_proxy: "{{ groups['maas_pacemaker'] | join(',') }},{{ maas_pacemaker_noproxy_list }},{{ maas_postgres_floating_ip }}"
+    NO_PROXY: "{{ groups['maas_pacemaker'] | join(',') }},{{ maas_pacemaker_noproxy_list }},{{ maas_postgres_floating_ip }}"
 
 - name: "Add Pacemaker Configuration Script"
   ansible.builtin.template:
@@ -85,6 +97,11 @@
     removes: "{{ maas_pacemaker_tmp_cib }}"
   run_once: true
 
+- name: "Configure HA metrics agent"
+  ansible.builtin.include_tasks:
+    file: o11y.yaml
+  when: o11y_enable
+
 - name: "Setup firewall"
   ansible.builtin.include_role:
     name: maas_firewall

roles/maas_pacemaker/tasks/o11y.yaml

@@ -0,0 +1,38 @@
+---
+- name: "Create ha_cluster_exporter directory"
+  ansible.builtin.file:
+    path: "{{ ha_exp_dir }}"
+    state: "directory"
+    mode: 0755
+
+- name: "Download latest ha_cluster_exporter release"
+  ansible.builtin.get_url:
+    url: "{{ ha_exp_pkg }}"
+    dest: "{{ ha_exp_dir ~ '/ha_cluster_exporter.gz' }}"
+
+- name: "Extract ha_cluster_exporter"
+  ansible.builtin.command:
+    cmd: gunzip "{{ ha_exp_dir }}/ha_cluster_exporter.gz"
+    creates: "{{ ha_exp_dir ~ '/ha_cluster_exporter' }}"
+
+- name: "Make agent executable"
+  ansible.builtin.file:
+    path: "{{ ha_exp_dir ~ '/ha_cluster_exporter' }}"
+    mode: 0755
+
+- name: "Create a service file"
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/ha_cluster_exporter.service
+    mode: 0644
+    content: |
+      [Unit]
+      Description=Prometheus exporter for Pacemaker HA clusters metrics
+      After=network.target
+      [Service]
+      Type=simple
+      ExecStart={{ ha_exp_dir }}/ha_cluster_exporter $ARGS
+      ExecReload=/bin/kill -HUP $MAINPID
+      Restart=always
+      [Install]
+      WantedBy=multi-user.target
+  notify: "Start HA clusters metrics agent"

roles/maas_pacemaker/tasks/teardown.yaml

@@ -1,4 +1,9 @@
 ---
+- name: "Stop Cluster services"
+  ansible.builtin.command: pcs cluster stop --all
+  changed_when: false
+  when: "'pacemaker' in ansible_facts.packages"
+
 - name: "Uninstall Pacemaker Packages"
   ansible.builtin.apt:
     name:
@@ -20,4 +25,6 @@
     - /tmp/pacemaker_auth
     - /etc/tmpfiles.d/postgresql-part.conf
     - /etc/ssh/sshd_config.d/pacemaker_sshd.conf
-    - "{{ maas_pacemaker_tmp_cib }}"
+    - "{{ maas_pacemaker_tmp_cib|d(false) }}"
+    - /etc/systemd/system/ha_cluster_exporter.service
+    - "{{ ha_exp_dir }}"

roles/maas_postgres/handlers/main.yaml

@@ -17,4 +17,10 @@
     name: "xinetd.service"
     state: restarted
     enabled: true
-  when: maas_ha_postgres_enabled|bool
+
+- name: "Start Postgres metrics agent"
+  ansible.builtin.systemd:
+    name: postgres_exporter
+    enabled: true
+    state: "restarted"
+    daemon-reload: true

roles/maas_postgres/tasks/main.yaml

@@ -13,6 +13,11 @@
     file: configure_postgres_secondary.yaml
   when: maas_ha_postgres_enabled|bool
 
+- name: "Configure Postgres metrics agent"
+  ansible.builtin.include_tasks:
+    file: o11y_pg.yaml
+  when: o11y_enable
+
 - name: "Setup firewall"
   ansible.builtin.include_role:
     name: maas_firewall

roles/maas_postgres/tasks/o11y_pg.yaml

@@ -2,22 +2,29 @@
 - name: "Create postgres-exporter directory"
   ansible.builtin.file:
     path: "{{ o11y_postgres_exporter_dir }}"
-    state: "{{ 'directory' if (o11y_enable and 'maas_postgres' in group_names) else 'absent' }}"
+    state: "directory"
     mode: 0755
-  changed_when: false
+
+- name: Install unzip
+  ansible.builtin.apt:
+    name: unzip
+    state: "present"
 
 - name: "Download and unzip latest postgres-exporter release"
   ansible.builtin.unarchive:
     src: "{{ pg_exp_pkg }}"
     dest: "{{ o11y_postgres_exporter_dir }}"
     remote_src: true
+    creates: "{{ o11y_postgres_exporter_dir ~ '/postgres_exporter' }}"
+    extra_opts:
+      - "--strip-components"
+      - "1"
 
 - name: "Set env variable for data source"
   ansible.builtin.copy:
+    dest: "{{ o11y_postgres_exporter_dir ~ '/postgres_exporter.env' }}"
     mode: 0644
-    dest: "{{ o11y_postgres_exporter_dir }}/postgres_exporter-0.11.1.linux-{{ ubuntu_arch }}/postgres_exporter.env"
-    content: |
-      DATA_SOURCE_NAME="postgresql://{{ maas_postgres_user }}:{{ maas_postgres_password }}@{{ inventory_hostname }}:5432/?sslmode=disable"
+    content: "{{ pg_exp_conn }}"
 
 - name: "Create a service file"
   ansible.builtin.copy:
@@ -32,23 +39,10 @@
       User=postgres
       Group=postgres
       Type=simple
-      EnvironmentFile={{ o11y_postgres_exporter_dir }}/postgres_exporter-0.11.1.linux-{{ ubuntu_arch }}/postgres_exporter.env
-      ExecStart={{ o11y_postgres_exporter_dir }}/postgres_exporter-0.11.1.linux-{{ ubuntu_arch }}/postgres_exporter
+      EnvironmentFile={{ o11y_postgres_exporter_dir }}/postgres_exporter.env
+      ExecStart={{ o11y_postgres_exporter_dir }}/postgres_exporter $ARGS
+      ExecReload=/bin/kill -HUP $MAINPID
       Restart=always
       [Install]
       WantedBy=multi-user.target
-  when: o11y_enable
-
-- name: "Reload systemd to re-read configurations"
-  ansible.builtin.systemd:
-    daemon-reload: true
-
-- name: "Collect facts about system services"
-  ansible.builtin.service_facts:
-
-- name: "Set the pg-exporter agent service status"
-  ansible.builtin.service:
-    name: postgres_exporter
-    enabled: "{{ o11y_enable }}"
-    state: "{{ 'started' if o11y_enable else 'stopped' }}"
-  when: ansible_facts.services['postgres_exporter.service'] is defined
+  notify: "Start Postgres metrics agent"

roles/maas_postgres/tasks/teardown.yaml

@@ -7,13 +7,21 @@
   ansible.builtin.set_fact:
     maas_postgres_config_dir: "/etc/postgresql/{{ maas_postgres_version_number }}/main/"
 
+- name: Stop Postgres metrics agent
+  ansible.builtin.service:
+    name: postgres_exporter
+    state: stopped
+  when: ansible_facts.services['postgres_exporter.service'] is defined
+
 - name: "Remove PostgreSQL artifacts"
   ansible.builtin.file:
     state: absent
     name: "{{ item }}"
   with_items:
     - /opt/pgsql_check
     - /etc/xinetd.d/pgsql_check
+    - /etc/systemd/system/postgres_exporter.service
+    - "{{ o11y_postgres_exporter_dir }}"
 
 - name: "Uninstall PostgreSQL and configuration dependencies"
   ansible.builtin.apt:

roles/o11y_agent/handlers/main.yaml

@@ -0,0 +1,7 @@
+---
+- name: "Start grafana agent"
+  ansible.builtin.systemd:
+    name: telemetry
+    enabled: true
+    state: "restarted"
+    daemon-reload: true