ceph-exporter: add installation role

group_vars/all.yml.sample

@@ -541,8 +541,6 @@ dummy:
 #    CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
 #    TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
 #  args:
-#    - --setuser=ceph
-#    - --setgroup=ceph
 #    - --default-log-to-file=false
 #    - --default-log-to-stderr=true
 #    - --default-log-stderr-prefix="debug "

group_vars/exporters.yml.sample

@@ -0,0 +1,30 @@
+---
+# Variables here are applicable to all host groups NOT roles
+
+# This sample file generated by generate_group_vars_sample.sh
+
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+###########
+# GENERAL #
+###########
+
+#ceph_exporter_addr: "0.0.0.0"
+#ceph_exporter_port: 9926
+#ceph_exporter_stats_period: 5  # seconds
+
+##########
+# DOCKER #
+##########
+
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+#ceph_exporter_container_params:
+#  args:
+#    - -f
+#    - --sock-dir=/var/run/ceph
+#    - --addrs={{ ceph_exporter_addr }}
+#    - --port={{ ceph_exporter_port }}
+#    - --stats-period={{ ceph_exporter_stats_period }}
+

group_vars/mdss.yml.sample

@@ -26,7 +26,6 @@ dummy:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_mds_docker_extra_env' variable.
 #ceph_mds_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 #ceph_mds_docker_cpu_limit: 4
 

infrastructure-playbooks/add-mon.yml

@@ -101,6 +101,11 @@
         name: ceph-crash
       when: containerized_deployment | bool
 
+    - name: Import ceph-exporter role
+      ansible.builtin.import_role:
+        name: ceph-exporter
+      when: containerized_deployment | bool
+
 - name: Update config file on OSD nodes
   hosts: osds
   gather_facts: true

infrastructure-playbooks/cephadm-adopt.yml

@@ -1187,6 +1187,43 @@
         CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'
 
 
+- name: Redeploy ceph-exporter daemons
+  hosts:
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
+    - "{{ mgr_group_name|default('mgrs') }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+  become: true
+  gather_facts: false
+  any_errors_fatal: true
+  tasks:
+    - name: Import ceph-defaults role
+      ansible.builtin.import_role:
+        name: ceph-defaults
+
+    - name: Stop and disable ceph-exporter systemd service
+      ansible.builtin.service:
+        name: "{{ 'ceph-exporter@' + ansible_facts['hostname'] if containerized_deployment | bool else 'ceph-exporter.service' }}"
+        state: stopped
+        enabled: false
+      failed_when: false
+
+    - name: Remove ceph-exporter systemd unit file
+      ansible.builtin.file:
+        path: /etc/systemd/system/[email protected]
+        state: absent
+
+    - name: Update the placement of ceph-exporter hosts
+      ansible.builtin.command: "{{ cephadm_cmd }} shell -k /etc/ceph/{{ cluster }}.client.admin.keyring --fsid {{ fsid }} -- ceph orch apply ceph-exporter --placement='label:ceph'"
+      run_once: true
+      changed_when: false
+      delegate_to: '{{ groups[mon_group_name][0] }}'
+      environment:
+        CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'
+
+
 - name: Redeploy alertmanager/grafana/prometheus daemons
   hosts: "{{ monitoring_group_name|default('monitoring') }}"
   serial: 1

infrastructure-playbooks/cephadm.yml

@@ -312,6 +312,12 @@
       environment:
         CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'
 
+    - name: Update the placement of ceph-exporter hosts
+      ansible.builtin.command: "{{ cephadm_cmd }} shell -- ceph --cluster {{ cluster }} orch apply ceph-exporter --placement='label:ceph'"
+      changed_when: false
+      environment:
+        CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'
+
 - name: Adjust monitoring service placement
   hosts: "{{ monitoring_group_name|default('monitoring') }}"
   become: true

infrastructure-playbooks/docker-to-podman.yml

@@ -194,6 +194,17 @@
             inventory_hostname in groups.get(mgr_group_name, []) or
             inventory_hostname in groups.get(rbdmirror_group_name, [])
 
+    - name: Import ceph-exporter role
+      ansible.builtin.import_role:
+        name: ceph-exporter
+        tasks_from: systemd.yml
+      when: inventory_hostname in groups.get(mon_group_name, []) or
+            inventory_hostname in groups.get(osd_group_name, []) or
+            inventory_hostname in groups.get(mds_group_name, []) or
+            inventory_hostname in groups.get(rgw_group_name, []) or
+            inventory_hostname in groups.get(mgr_group_name, []) or
+            inventory_hostname in groups.get(rbdmirror_group_name, [])
+
     - name: Dashboard configuration
       when: dashboard_enabled | bool
       block:

infrastructure-playbooks/purge-cluster.yml

@@ -726,6 +726,40 @@
         state: absent
 
 
+- name: Purge ceph-exporter daemons
+  hosts:
+    - mons
+    - osds
+    - mdss
+    - rgws
+    - rbdmirrors
+    - mgrs
+  gather_facts: false
+  become: true
+  tasks:
+    - name: Import ceph-defaults role
+      ansible.builtin.import_role:
+        name: ceph-defaults
+
+    - name: Stop ceph-exporter service
+      ansible.builtin.service:
+        name: "{{ 'ceph-exporter@' + ansible_facts['hostname'] if containerized_deployment | bool else 'ceph-exporter.service' }}"
+        state: stopped
+        enabled: false
+      failed_when: false
+
+    - name: Systemctl reset-failed ceph-exporter  # noqa command-instead-of-module
+      ansible.builtin.command: "systemctl reset-failed {{ 'ceph-exporter@' + ansible_facts['hostname'] if containerized_deployment | bool else 'ceph-exporter.service' }}"
+      changed_when: false
+      failed_when: false
+
+    - name: Remove service file
+      ansible.builtin.file:
+        name: "/etc/systemd/system/ceph-exporter{{ '@' if containerized_deployment | bool else '' }}.service"
+        state: absent
+      failed_when: false
+
+
 - name: Check container hosts
   hosts:
     - mons

infrastructure-playbooks/rolling_update.yml

@@ -1143,6 +1143,50 @@
       ansible.builtin.import_role:
         name: ceph-crash
 
+- name: Upgrade ceph-exporter daemons
+  hosts:
+    - "{{ mon_group_name | default('mons') }}"
+    - "{{ osd_group_name | default('osds') }}"
+    - "{{ mds_group_name | default('mdss') }}"
+    - "{{ rgw_group_name | default('rgws') }}"
+    - "{{ rbdmirror_group_name | default('rbdmirrors') }}"
+    - "{{ mgr_group_name | default('mgrs') }}"
+  tags:
+    - post_upgrade
+    - ceph-exporter
+  gather_facts: false
+  become: true
+  tasks:
+    - name: Exit ceph-exporter upgrade if non containerized deployment
+      ansible.builtin.meta: end_play
+      when: not containerized_deployment | bool
+
+    - name: Stop the ceph-exporter service
+      ansible.builtin.systemd:
+        name: "{{ 'ceph-exporter@' + ansible_facts['hostname'] if containerized_deployment | bool else 'ceph-exporter.service' }}"
+        state: stopped
+
+    # it needs to be done in a separate task otherwise the stop just before doesn't work.
+    - name: Mask and disable the ceph-exporter service
+      ansible.builtin.systemd:
+        name: "{{ 'ceph-exporter@' + ansible_facts['hostname'] if containerized_deployment | bool else 'ceph-exporter.service' }}"
+        enabled: false
+        masked: true
+
+    - name: Import ceph-defaults role
+      ansible.builtin.import_role:
+        name: ceph-defaults
+    - name: Import ceph-facts role
+      ansible.builtin.import_role:
+        name: ceph-facts
+        tasks_from: container_binary.yml
+    - name: Import ceph-handler role
+      ansible.builtin.import_role:
+        name: ceph-handler
+    - name: Import ceph-exporter role
+      ansible.builtin.import_role:
+        name: ceph-exporter
+
 - name: Complete upgrade
   hosts: "{{ mon_group_name | default('mons') }}"
   tags: post_upgrade

infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml

@@ -755,6 +755,37 @@
       ansible.builtin.import_role:
         name: ceph-crash
 
+- name: Switching from non-containerized to containerized ceph-exporter
+
+  hosts:
+    - "{{ mon_group_name | default('mons') }}"
+    - "{{ osd_group_name | default('osds') }}"
+    - "{{ mds_group_name | default('mdss') }}"
+    - "{{ rgw_group_name | default('rgws') }}"
+    - "{{ rbdmirror_group_name | default('rbdmirrors') }}"
+    - "{{ mgr_group_name | default('mgrs') }}"
+
+  vars:
+    containerized_deployment: true
+  become: true
+  tasks:
+    - name: Import ceph-defaults role
+      ansible.builtin.import_role:
+        name: ceph-defaults
+
+    - name: Import ceph-facts role
+      ansible.builtin.import_role:
+        name: ceph-facts
+        tasks_from: container_binary.yml
+
+    - name: Import ceph-handler role
+      ansible.builtin.import_role:
+        name: ceph-handler
+
+    - name: Import ceph-exporter role
+      ansible.builtin.import_role:
+        name: ceph-exporter
+
 - name: Final task
   hosts:
     - "{{ mon_group_name|default('mons') }}"

plugins/callback/installer_checkpoint.py

@@ -34,6 +34,7 @@ def v2_playbook_on_stats(self, stats):
             'installer_phase_ceph_grafana',
             'installer_phase_ceph_node_exporter',
             'installer_phase_ceph_crash',
+            'installer_phase_ceph_exporter',
         ]
 
         # Define the attributes of the installer phases
@@ -90,6 +91,10 @@ def v2_playbook_on_stats(self, stats):
                 'title': 'Install Ceph Crash',
                 'playbook': 'roles/ceph-crash/tasks/main.yml'
             },
+            'installer_phase_ceph_exporter': {
+                'title': 'Install Ceph Exporter',
+                'playbook': 'roles/ceph-exporter/tasks/main.yml'
+            },
         }
 
         # Find the longest phase title

roles/ceph-exporter/defaults/main.yml

@@ -0,0 +1,24 @@
+---
+###########
+# GENERAL #
+###########
+
+ceph_exporter_addr: "0.0.0.0"
+ceph_exporter_port: 9926
+ceph_exporter_stats_period: 5  # seconds
+ceph_exporter_prio_limit: 5
+
+##########
+# DOCKER #
+##########
+
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+ceph_exporter_container_params:
+  args:
+    - -f
+    - -n=client.ceph-exporter
+    - --sock-dir=/var/run/ceph
+    - --addrs={{ ceph_exporter_addr }}
+    - --port={{ ceph_exporter_port }}
+    - --stats-period={{ ceph_exporter_stats_period }}
+    - --prio-limit={{ ceph_exporter_prio_limit }}

roles/ceph-exporter/meta/main.yml

@@ -0,0 +1,14 @@
+---
+galaxy_info:
+  company: Red Hat
+  author: Guillaume Abrioux
+  description: Deploy ceph-exporter
+  license: Apache
+  min_ansible_version: '2.7'
+  platforms:
+    - name: EL
+      versions:
+        - 'all'
+  galaxy_tags:
+    - system
+dependencies: []

roles/ceph-exporter/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: Create and copy client.ceph-exporter keyring
+  when: cephx | bool
+  block:
+    - name: Create client.ceph-exporter keyring
+      ceph_key:
+        name: "client.ceph-exporter"
+        caps:
+          mon: 'allow r'
+          mgr: 'allow r'
+          osd: 'allow r'
+        cluster: "{{ cluster }}"
+        dest: "{{ ceph_conf_key_directory }}"
+        import_key: true
+        mode: "{{ ceph_keyring_permissions }}"
+        owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      delegate_to: "{{ groups.get(mon_group_name, [])[0] }}"
+      run_once: true
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+
+    - name: Get keys from monitors
+      ceph_key:
+        name: client.ceph-exporter
+        cluster: "{{ cluster }}"
+        output_format: plain
+        state: info
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      register: _exporter_keys
+      delegate_to: "{{ groups.get(mon_group_name)[0] }}"
+      run_once: true
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+
+    - name: Copy ceph key(s) if needed
+      ansible.builtin.copy:
+        dest: "{{ ceph_conf_key_directory }}/{{ cluster }}.client.ceph-exporter.keyring"
+        content: "{{ _exporter_keys.stdout + '\n' }}"
+        owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+        mode: "{{ ceph_keyring_permissions }}"
+      no_log: "{{ no_log_on_ceph_key_tasks }}"
+
+- name: Include_tasks systemd.yml
+  ansible.builtin.include_tasks: systemd.yml
+  when: containerized_deployment | bool
+
+- name: Start the ceph-exporter service
+  ansible.builtin.systemd:
+    name: "{{ 'ceph-exporter@' + ansible_facts['hostname'] if containerized_deployment | bool else 'ceph-exporter.service' }}"
+    state: started
+    enabled: true
+    masked: false
+    daemon_reload: true

roles/ceph-exporter/tasks/systemd.yml

@@ -0,0 +1,9 @@
+---
+- name: Generate systemd unit file for ceph-exporter container
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/ceph-exporter.service.j2"
+    dest: /etc/systemd/system/[email protected]
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  notify: Restart ceph exporter