From 82e57052d8a0406a5ce65ed34b7c525da6b55f2a Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 08:26:44 +0000 Subject: [PATCH 01/35] PMM-12693 Run grafana as non-root --- .../roles/pmm-images/files/grafana.ini | 2 +- build/packages/rpm/server/SPECS/grafana.spec | 26 +++++++++-------- managed/services/supervisord/supervisord.go | 2 +- managed/testdata/supervisord.d/grafana.ini | 2 +- .../tasks/roles/grafana/tasks/main.yml | 29 ++++++++++++------- 5 files changed, 36 insertions(+), 25 deletions(-) diff --git a/build/ansible/roles/pmm-images/files/grafana.ini b/build/ansible/roles/pmm-images/files/grafana.ini index 0e8b05106c..8bc767213d 100644 --- a/build/ansible/roles/pmm-images/files/grafana.ini +++ b/build/ansible/roles/pmm-images/files/grafana.ini @@ -7,7 +7,7 @@ command = /usr/sbin/grafana server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini -user = grafana +user = pmm directory = /usr/share/grafana autorestart = true autostart = true diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec index 43f815c837..70c12653f6 100644 --- a/build/packages/rpm/server/SPECS/grafana.spec +++ b/build/packages/rpm/server/SPECS/grafana.spec @@ -2,7 +2,7 @@ %global commit 7ff49f34a3998067fa1ea480c07e0c74939ea306 %global shortcommit %(c=%{commit}; echo ${c:0:7}) %define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 100 +%define release 101 %define grafana_version 9.2.20 %define full_pmm_version 2.0.0 %define full_version v%{grafana_version}-%{full_pmm_version} @@ -48,10 +48,11 @@ make deps-js make build-js %install +%defattr(-, pmm, pmm, -) install -d -p %{buildroot}%{_datadir}/grafana -cp -rpav conf %{buildroot}%{_datadir}/grafana -cp -rpav public %{buildroot}%{_datadir}/grafana -cp -rpav tools %{buildroot}%{_datadir}/grafana +cp -rav conf %{buildroot}%{_datadir}/grafana +cp -rav public %{buildroot}%{_datadir}/grafana +cp -rav tools %{buildroot}%{_datadir}/grafana install -d -p %{buildroot}%{_sbindir} cp bin/linux-amd64/grafana-server %{buildroot}%{_sbindir}/ @@ -65,25 +66,26 @@ mv conf/ldap.toml %{buildroot}%{_sysconfdir}/grafana/ install -d -p %{buildroot}%{_sharedstatedir}/grafana %files -%defattr(-, grafana, grafana, -) +%defattr(-, pmm, pmm, -) %{_datadir}/grafana %doc CHANGELOG.md README.md %license LICENSE -%attr(0755, root, root) %{_sbindir}/grafana -%attr(0755, root, root) %{_sbindir}/grafana-server -%attr(0755, root, root) %{_bindir}/grafana-cli +%attr(0755, pmm, pmm) %{_sbindir}/grafana +%attr(0755, pmm, pmm) %{_sbindir}/grafana-server +%attr(0755, pmm, pmm) %{_bindir}/grafana-cli %{_sysconfdir}/grafana/grafana.ini %{_sysconfdir}/grafana/ldap.toml %dir %{_sharedstatedir}/grafana %pre -getent group grafana >/dev/null || groupadd -r grafana -getent passwd grafana >/dev/null || \ - useradd -r -g grafana -d /etc/grafana -s /sbin/nologin \ - -c "Grafana Server" grafana +getent group pmm >/dev/null || echo "Group pmm does not exist. Please create it manually." +getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it manually." exit 0 %changelog +* Mon Nov 27 2023 Alex Demidoff - 9.2.20-2 +- PMM-12693 Run Grafana as non-root user + * Tue Jun 27 2023 Matej Kubinec - 9.2.20-1 - PMM-12254 Grafana 9.2.20 diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 79cfede3c5..756536da9d 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -800,7 +800,7 @@ environment = {{- if .PerconaSSODetails}} GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} -user = grafana +user = pmm directory = /usr/share/grafana autorestart = true autostart = true diff --git a/managed/testdata/supervisord.d/grafana.ini b/managed/testdata/supervisord.d/grafana.ini index c6a0788699..be7dfec97f 100644 --- a/managed/testdata/supervisord.d/grafana.ini +++ b/managed/testdata/supervisord.d/grafana.ini @@ -18,7 +18,7 @@ environment = PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE_ADDR="127.0.0.1:8123", PERCONA_TEST_PMM_CLICKHOUSE_HOST="127.0.0.1", PERCONA_TEST_PMM_CLICKHOUSE_PORT="9000", -user = grafana +user = pmm directory = /usr/share/grafana autorestart = true autostart = true diff --git a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml index e212ed109d..50d1c2f5d0 100644 --- a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml @@ -3,9 +3,9 @@ file: path: "{{ item }}" state: directory - owner: grafana - group: grafana - mode: "0775" + owner: pmm + group: pmm + mode: 0775 loop: - /srv/grafana - /srv/grafana/plugins @@ -15,13 +15,22 @@ path: /etc/bashrc line: "export GF_PLUGIN_DIR=/srv/grafana/plugins" +- name: Set ownership on /etc/grafana + file: + path: /etc/grafana + state: directory + owner: pmm + group: pmm + mode: 0644 + recurse: true + - name: Copy new version of grafana.ini copy: src: grafana.ini dest: /etc/grafana/grafana.ini - owner: grafana - group: grafana - mode: "0444" + owner: pmm + group: pmm + mode: 0444 - name: Check that the SQLite grafana database exists stat: @@ -62,8 +71,8 @@ - name: Create provisioning directory file: path: "/usr/share/grafana/conf/provisioning/{{ item }}" - owner: grafana - group: grafana + owner: pmm + group: pmm state: directory loop: - datasources @@ -74,8 +83,8 @@ copy: src: "{{ item }}.yml" dest: "/usr/share/grafana/conf/provisioning/{{ item }}/default.yml" - owner: grafana - group: grafana + owner: pmm + group: pmm loop: - datasources - plugins From 8346c9286dd7a73a5f926419e9860267a7eebb62 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 09:52:51 +0000 Subject: [PATCH 02/35] PMM-12693 rebuild grafana with new perms --- build/packages/rpm/server/SPECS/grafana.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec index 70c12653f6..80a36b85b6 100644 --- a/build/packages/rpm/server/SPECS/grafana.spec +++ b/build/packages/rpm/server/SPECS/grafana.spec @@ -2,7 +2,7 @@ %global commit 7ff49f34a3998067fa1ea480c07e0c74939ea306 %global shortcommit %(c=%{commit}; echo ${c:0:7}) %define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 101 +%define release 102 %define grafana_version 9.2.20 %define full_pmm_version 2.0.0 %define full_version v%{grafana_version}-%{full_pmm_version} From 65b686122fefc92b1e87546d395f67bce3231c83 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 12:13:21 +0000 Subject: [PATCH 03/35] PMM-12693 remove sqlite migration --- .../tasks/roles/initialization/tasks/main.yml | 70 +------ .../roles/sqlite-to-postgres/tasks/main.yml | 177 ------------------ update/ansible/playbook/tasks/update.yml | 18 +- 3 files changed, 11 insertions(+), 254 deletions(-) delete mode 100644 update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 021b246b2b..33a42cdcd5 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -108,69 +108,13 @@ - name: Create empty configuration file for VictoriaMetrics file: path=/etc/victoriametrics-promscrape.yml state=touch owner=pmm group=pmm -- name: Run operations for docker-way upgrade - block: - - name: Check that the SQLite grafana database exists - stat: - path: /srv/grafana/grafana.db - register: is_database_sqlite - - - name: Run SQLite -> Postgres only - block: - - name: Remove database options (SQLite is default) - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: type - state: absent - - - name: Remove database host - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: host - state: absent - - - name: Remove database user - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: user - state: absent - - - name: Remove database password - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: password - state: absent - - - name: Upgrade grafana database (Get the latest schema) - command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords - changed_when: True - - - name: Start grafana again - supervisorctl: - name: grafana - state: restarted - - - name: Wait for grafana - pause: seconds=10 - - - name: Migrate Grafana database from SQLite to Postgresql - include_role: - name: sqlite-to-postgres - tags: - - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. - when: is_database_sqlite.stat.exists - - - name: Change default admin id - postgresql_query: - db: grafana - query: UPDATE "user" SET id='1' WHERE login='admin'; - when: not ansible_check_mode - - when: docker_upgrade +- name: Change default admin id + postgresql_query: + db: grafana + query: UPDATE "user" SET id='1' WHERE login='admin'; + when: + - docker_upgrade + - not ansible_check_mode - name: Check if we need an update or not include_role: diff --git a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml b/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml deleted file mode 100644 index 8a905dcf3e..0000000000 --- a/update/ansible/playbook/tasks/roles/sqlite-to-postgres/tasks/main.yml +++ /dev/null @@ -1,177 +0,0 @@ ---- -- name: Wait for grafana to update DB and be ready - ansible.builtin.uri: - url: "http://127.0.0.1:3000/api/health" - status_code: 200 - method: GET - register: healthcheck - until: healthcheck is not failed - retries: 120 - delay: 1 - -- name: Create Grafana backup dir - file: - path: "/srv/backup/grafana" - state: directory - owner: grafana - group: grafana - mode: '0700' - -- name: Stop grafana before upgrade - supervisorctl: - name: 'grafana' - state: stopped - -- name: Drop grafana database from postgres EL7 - command: dropdb -f grafana -U postgres - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - - not ansible_check_mode - -- name: Drop grafana database from postgres EL9 - postgresql_db: - name: grafana - state: absent - force: yes - when: - - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - - ansible_distribution_major_version == '9' - -- name: Create grafana database in postgres - postgresql_db: - name: grafana - state: present - -- name: Create grafana user in postgres - postgresql_user: - db: grafana - name: grafana - password: grafana - priv: 'ALL' - expires: infinity - state: present - when: not ansible_check_mode - -- name: Create backup for SQLite Grafana database - copy: - src: /srv/grafana/grafana.db - dest: "/srv/backup/grafana/grafana.db" - owner: grafana - group: grafana - mode: '0700' - -- name: Remove all ` symbols in grafana dashboard description - command: sqlite3 /srv/grafana/grafana.db -cmd ".timeout 60000" "UPDATE dashboard SET data = REPLACE(data, '`', '');" - changed_when: True - -- name: Disable provisioning before change database - ini_file: - dest: /etc/grafana/grafana.ini - section: paths - option: provisioning - value: conf/provisioning_disable - -- name: Switch to postgres - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: type - value: postgres - -- name: Set database host - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: host - value: localhost - -- name: Set database user - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: user - value: grafana - -- name: Set database password - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: password - value: grafana - -- name: Start grafana again - supervisorctl: - name: grafana - state: restarted - ignore_errors: yes - -- name: Check if initial data were created - postgresql_query: - db: grafana - query: SELECT 1 FROM org WHERE id=1 - retries: 3 - delay: 3 - register: psql_result - until: psql_result.rowcount == 1 - when: not ansible_check_mode - ignore_errors: yes - -- name: Wait for grafana database initialization - pause: - seconds: 10 - -- name: Stop grafana before upgrade - supervisorctl: - name: grafana - state: stopped - -- name: Remove default admin user - postgresql_query: - db: grafana - query: DELETE FROM public.user WHERE login='admin' - when: not ansible_check_mode - -- name: Run grafana migrator - command: grafana-db-migrator --change-char-to-text /srv/grafana/grafana.db "postgres://grafana:grafana@localhost:5432/grafana?sslmode=disable" - register: migrator_output - changed_when: "'All done' in migrator_output.stdout" - -- name: copy the output of grafana-db-migrator to file - copy: - content: "{{ migrator_output.stdout }}" - dest: "/srv/logs/grafana-db-migrator.log" - when: not ansible_check_mode - -- name: Enable provisioning after change database - ini_file: - dest: /etc/grafana/grafana.ini - section: paths - option: provisioning - value: conf/provisioning - -- name: Start grafana again - supervisorctl: - name: grafana - state: restarted - -- name: Wait for grafana initialization - pause: - seconds: 5 - -- name: Fix database/folder relationship - command: grafana-db-migrator --fix-folders-id /srv/grafana/grafana.db "postgres://grafana:grafana@localhost:5432/grafana?sslmode=disable" - register: migrator_output - changed_when: "'All done' in migrator_output.stdout" - -- name: Copy the output of grafana-db-migrator to file - ansible.builtin.blockinfile: - dest: /srv/logs/grafana-db-migrator.log - block: "{{ migrator_output.stdout }}" - backup: yes - when: not ansible_check_mode - -- name: Remove SQLite Grafana database - file: - path: /srv/grafana/grafana.db - state: absent diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 4d25db6e61..474727c525 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -210,8 +210,7 @@ - name: Reread supervisord configuration EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - command: /usr/local/bin/supervisorctl reread - become: true + command: supervisorctl reread register: reread_result changed_when: "'No config updates to processes' not in reread_result.stdout" @@ -374,6 +373,9 @@ - pmm-agent - ["stop", "remove", "add"] + - name: Check supervisord logs + shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log + - name: Delete old grafana and yum-cron artifacts file: state: absent @@ -384,18 +386,6 @@ - /etc/yum/yum-cron-hourly.conf - /etc/yum/yum-cron.conf - - name: Check that the SQLite grafana database exists - stat: - path: /srv/grafana/grafana.db - register: is_database_sqlite - - - name: Migrate Grafana database from SQLite to Postgresql - include_role: - name: sqlite-to-postgres - when: is_database_sqlite.stat.exists - tags: - - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. - - name: Fix grafana fields type postgresql_query: db: grafana From 6c03ba9e8331f439a3dbd51cb67a67a49d2da5e8 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 12:43:43 +0000 Subject: [PATCH 04/35] PMM-12693 remove sqlite migration --- .../roles/dashboards_upgrade/tasks/main.yml | 5 --- .../tasks/roles/grafana/tasks/main.yml | 36 ------------------- update/ansible/playbook/tasks/update.yml | 5 +-- 3 files changed, 1 insertion(+), 45 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml index c9d4ae9019..7cd458b17d 100644 --- a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml @@ -58,11 +58,6 @@ group: grafana mode: "0775" -- name: Check that the SQLite grafana database exists - stat: - path: /srv/grafana/grafana.db - register: sqlite_grafana - - name: Remove the old clickhouse plugin shell: grafana cli --pluginsDir /srv/grafana/plugins plugins remove vertamedia-clickhouse-datasource || true diff --git a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml index 50d1c2f5d0..eae520f2a4 100644 --- a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml @@ -32,42 +32,6 @@ group: pmm mode: 0444 -- name: Check that the SQLite grafana database exists - stat: - path: /srv/grafana/grafana.db - register: sqlite_grafana - -- name: Temporary change database to SQLite - block: - - name: Remove database options (SQLite is default) - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: type - state: absent - - - name: Remove database host - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: host - state: absent - - - name: Remove database user - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: user - state: absent - - - name: Remove database password - ini_file: - dest: /etc/grafana/grafana.ini - section: database - option: password - state: absent - when: sqlite_grafana.stat.exists - - name: Create provisioning directory file: path: "/usr/share/grafana/conf/provisioning/{{ item }}" diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 474727c525..0ca502bd2f 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -376,15 +376,12 @@ - name: Check supervisord logs shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log - - name: Delete old grafana and yum-cron artifacts + - name: Delete unused grafana artifacts file: state: absent path: "{{ item }}" loop: - - /var/lib/grafana/plugins/ - /usr/share/grafana/data/grafana.db - - /etc/yum/yum-cron-hourly.conf - - /etc/yum/yum-cron.conf - name: Fix grafana fields type postgresql_query: From da21c09caa8cc636133c0ab3ce6b01902a7f2a94 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 13:08:11 +0000 Subject: [PATCH 05/35] PMM-12693 revert some spec changes --- build/packages/rpm/server/SPECS/grafana.spec | 7 +++---- .../tasks/roles/dashboards_upgrade/tasks/main.yml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec index 80a36b85b6..f3d6e644ee 100644 --- a/build/packages/rpm/server/SPECS/grafana.spec +++ b/build/packages/rpm/server/SPECS/grafana.spec @@ -48,11 +48,10 @@ make deps-js make build-js %install -%defattr(-, pmm, pmm, -) install -d -p %{buildroot}%{_datadir}/grafana -cp -rav conf %{buildroot}%{_datadir}/grafana -cp -rav public %{buildroot}%{_datadir}/grafana -cp -rav tools %{buildroot}%{_datadir}/grafana +cp -rpav conf %{buildroot}%{_datadir}/grafana +cp -rpav public %{buildroot}%{_datadir}/grafana +cp -rpav tools %{buildroot}%{_datadir}/grafana install -d -p %{buildroot}%{_sbindir} cp bin/linux-amd64/grafana-server %{buildroot}%{_sbindir}/ diff --git a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml index 7cd458b17d..8db53ed578 100644 --- a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml @@ -54,9 +54,10 @@ file: path: "/srv/grafana/plugins" state: directory - owner: grafana - group: grafana - mode: "0775" + owner: pmm + group: pmm + mode: 0775 + recurse: yes - name: Remove the old clickhouse plugin shell: grafana cli --pluginsDir /srv/grafana/plugins plugins remove vertamedia-clickhouse-datasource || true @@ -77,7 +78,9 @@ state: restarted supervisorctl_path: /usr/local/bin/supervisorctl become: true - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' ignore_errors: true # TODO: fix the race condition. # We generate grafana supervisor config in pmm-managed and it may not exist at this stage @@ -86,4 +89,7 @@ copy: src: /usr/share/percona-dashboards/VERSION dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION + owner: pmm + group: pmm + mode: 0444 remote_src: yes From a67b354c2a43ce60f5ee124f109c6ed176a51ce5 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 14:26:21 +0000 Subject: [PATCH 06/35] PMM-12693 check grafana logs --- update/ansible/playbook/tasks/update.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 0ca502bd2f..44d743cd0a 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -374,7 +374,10 @@ - ["stop", "remove", "add"] - name: Check supervisord logs - shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log + shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log + + - name: Check grafana logs + shell: tail -n 200 /srv/logs/grafana.log - name: Delete unused grafana artifacts file: From be227be908e7bc291b20d9c476c3c3c5bebf4d0e Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 16:19:48 +0000 Subject: [PATCH 07/35] PMM-12693 remove grafana-db-migrator --- build/ansible/roles/pmm-images/tasks/main.yml | 1 - build/scripts/build-server-rpm-all | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 0a96c7322d..4ea2b56eaa 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -158,7 +158,6 @@ - percona-alertmanager - pmm-managed - pmm-update - - grafana-db-migrator - pmm-dump - vmproxy state: installed diff --git a/build/scripts/build-server-rpm-all b/build/scripts/build-server-rpm-all index ab21a205ee..6d65e16600 100755 --- a/build/scripts/build-server-rpm-all +++ b/build/scripts/build-server-rpm-all @@ -10,13 +10,12 @@ ${bin_dir}/build-server-rpm pmm-managed pmm ${bin_dir}/build-server-rpm percona-qan-api2 pmm ${bin_dir}/build-server-rpm pmm-update pmm ${bin_dir}/build-server-rpm pmm-dump -${bin_dir}/build-server-rpm grafana-db-migrator +# ${bin_dir}/build-server-rpm grafana-db-migrator ${bin_dir}/build-server-rpm vmproxy pmm # 3rd-party ${bin_dir}/build-server-rpm victoriametrics ${bin_dir}/build-server-rpm alertmanager ${bin_dir}/build-server-rpm grafana -# ${bin_dir}/build-server-rpm grafana-db-migrator # vim: expandtab shiftwidth=4 tabstop=4 From c2f16822d0141a87be45be77b7ca6d17ce437c5a Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 16:37:53 +0000 Subject: [PATCH 08/35] PMM-12693 create grafana dir explicitly --- build/ansible/roles/pmm-images/tasks/main.yml | 19 +++- .../tasks/roles/grafana/tasks/main.yml | 2 +- update/ansible/playbook/tasks/update.yml | 105 +++--------------- 3 files changed, 29 insertions(+), 97 deletions(-) diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 4ea2b56eaa..71eb32d22e 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -101,7 +101,7 @@ loop: - { name: pmm, gid: 1000 } - { name: nginx, gid: 999 } - - { name: grafana, gid: 998 } + - { name: grafana, gid: 1002 } - { name: clickhouse, gid: 997 } - { name: pmm-agent, gid: 996 } @@ -117,17 +117,22 @@ loop: - { name: pmm, uid: 1000, comment: "PMM Server", shell: "/bin/false", home: "/home/pmm", group: pmm, } - { name: nginx, uid: 999, comment: "nginx user", shell: "/sbin/nologin", home: "/var/cache/nginx", group: nginx, } - - { name: grafana, uid: 998, comment: "Grafana Dashboard", shell: "/sbin/nologin", home: "/etc/grafana", group: grafana, } + - { name: grafana, uid: 1002, comment: "Grafana Server", shell: "/sbin/nologin", home: "/home/grafana", group: grafana, } - { name: clickhouse, uid: 997, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/var/lib/clickhouse", group: clickhouse, } - { name: pmm-agent, uid: 996, comment: "pmm-agent", shell: "/bin/false", home: "/usr/local/percona/", group: pmm-agent, } when: ansible_virtualization_type == "docker" - name: Create directories | Create dirs - file: path={{ item }} state=directory owner=pmm group=pmm + file: + - path: "{{ item }}" + - state: directory + - owner: pmm + - group: pmm with_items: - /srv/prometheus/data - /srv/prometheus/rules - /srv/alertmanager/data + - /etc/grafana - name: Create directories | Create dirs file: @@ -135,11 +140,13 @@ state: directory owner: pmm group: pmm - mode: "0775" + mode: 0775 - name: Create dirs | Create dirs when: ansible_virtualization_type == "docker" - file: path={{ item }} state=directory + file: + - path: "{{ item }}" + - state: directory with_items: - /var/lib/cloud/scripts/per-once - /var/lib/cloud/scripts/per-boot @@ -186,4 +193,4 @@ copy: src: grafana.ini dest: /etc/supervisord.d/grafana.ini - mode: "0644" + mode: 0644 diff --git a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml index eae520f2a4..2d92a6e3cf 100644 --- a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml @@ -30,7 +30,7 @@ dest: /etc/grafana/grafana.ini owner: pmm group: pmm - mode: 0444 + mode: 0644 - name: Create provisioning directory file: diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 44d743cd0a..c2829cf907 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -18,7 +18,6 @@ - pmm-client - pmm-dump - vmproxy - - grafana-db-migrator pre_tasks: - name: detect /srv/pmm-distribution stat: @@ -165,12 +164,12 @@ path: /var/run/supervisor/supervisor.sock register: is_supervisor_running - - name: Supervisord start EL7 | Start supervisord for docker - when: is_docker and not is_supervisor_running.stat.exists and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - shell: supervisord -c /etc/supervisord.conf & - - name: Supervisord start EL9 | Start supervisord for docker - when: is_docker and not is_supervisor_running.stat.exists and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - is_docker + - not is_supervisor_running.stat.exists + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' shell: /usr/local/bin/supervisord -c /etc/supervisord.conf & - name: Wait until postgres port is present before continuing @@ -202,12 +201,6 @@ # See https://github.com/Supervisor/supervisor/issues/1264 for explanation # why we do reread + stop/remove/add instead of using supervisorctl Ansible module. - - name: Reread supervisord configuration EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl reread - register: reread_result - changed_when: "'No config updates to processes' not in reread_result.stdout" - - name: Reread supervisord configuration EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' command: supervisorctl reread @@ -217,16 +210,11 @@ - name: Check reread results debug: var=reread_result.stdout_lines - - name: Restart pmm-managed EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl {{ item }} pmm-managed - become: true - changed_when: true - with_items: ["stop", "remove", "add"] - - name: Restart pmm-managed EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - command: /usr/local/bin/supervisorctl {{ item }} pmm-managed + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + command: supervisorctl {{ item }} pmm-managed become: true changed_when: true with_items: ["stop", "remove", "add"] @@ -236,31 +224,6 @@ - name: Wait for pmm-managed pause: seconds=10 - - name: Update system packages EL7 - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - yum: - name: "*" - state: latest - security: yes - exclude: - - nginx* - - - name: Updating only select packages EL7 - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - yum: - name: "{{ item }}" - state: latest - loop: - - nss - - tzdata - - libssh2 - - sshpass - - vi - - name: Remove ansible RPM if installed | EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -332,37 +295,18 @@ regexp: "set -o errexit" replace: "" - - name: Reread supervisord configuration again EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl reread - register: reread_result - changed_when: "'No config updates to processes' not in reread_result.stdout" - - name: Reread supervisord configuration again EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - command: /usr/local/bin/supervisorctl reread + command: supervisorctl reread register: reread_result changed_when: "'No config updates to processes' not in reread_result.stdout" - name: Check reread results debug: var=reread_result.stdout_lines - - name: Restart services EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl {{ item.1 }} {{ item.0 }} - become: true - changed_when: true - with_nested: - - - alertmanager - - nginx - - grafana - - qan-api2 - - pmm-agent - - ["stop", "remove", "add"] - - name: Restart services EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' and is_docker - command: /usr/local/bin/supervisorctl {{ item.1 }} {{ item.0 }} + command: supervisorctl {{ item.1 }} {{ item.0 }} become: true changed_when: true with_nested: @@ -421,42 +365,23 @@ register: managed_init_result changed_when: True - - name: Reread pmm-update-perform-init supervisor config EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl reread - register: reread_init__result - changed_when: "'No config updates to processes' not in reread_init__result.stdout" - - name: Reread pmm-update-perform-init supervisor config EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - command: /usr/local/bin/supervisorctl reread + command: supervisorctl reread register: reread_init__result changed_when: "'No config updates to processes' not in reread_init__result.stdout" # restarting pmm-managed to regenerate /etc/alertmanager.yml - - name: Restart pmm-managed EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl {{ item }} pmm-managed - become: true - changed_when: true - with_items: ["stop", "remove", "add"] - - name: Restart pmm-managed EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - command: /usr/local/bin/supervisorctl {{ item }} pmm-managed + command: supervisorctl {{ item }} pmm-managed become: true changed_when: true with_items: ["stop", "remove", "add"] - - name: Update/restart other services EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl update - register: update_result - changed_when: "'updated' in update_result.stdout" - - name: Update/restart other services EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - command: /usr/local/bin/supervisorctl update + command: supervisorctl update register: update_result changed_when: "'updated' in update_result.stdout" @@ -483,7 +408,7 @@ - name: Get supervisord logs EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - shell: /usr/local/bin/supervisorctl maintail -100000 | tac | awk '!flag; /received SIGUSR2/{flag = 1};' | tac + shell: supervisorctl maintail -100000 | tac | awk '!flag; /received SIGUSR2/{flag = 1};' | tac register: maintail_result changed_when: False From a2cc6e40d0b92a21d0ee911fef4c28afb105d40e Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 17:25:06 +0000 Subject: [PATCH 09/35] PMM-12693 fix with_items syntax --- build/ansible/roles/pmm-images/tasks/main.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 71eb32d22e..4323111ca8 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -123,11 +123,7 @@ when: ansible_virtualization_type == "docker" - name: Create directories | Create dirs - file: - - path: "{{ item }}" - - state: directory - - owner: pmm - - group: pmm + file: path={{ item }} state=directory owner=pmm group=pmm with_items: - /srv/prometheus/data - /srv/prometheus/rules @@ -144,9 +140,7 @@ - name: Create dirs | Create dirs when: ansible_virtualization_type == "docker" - file: - - path: "{{ item }}" - - state: directory + file: path={{ item }} state=directory with_items: - /var/lib/cloud/scripts/per-once - /var/lib/cloud/scripts/per-boot From fc676aebcab3e7156fcd7dcf150297c7a1802a5a Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 18:04:02 +0000 Subject: [PATCH 10/35] PMM-12693 take ownership of /etc/grafana --- update/ansible/playbook/tasks/update.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index c2829cf907..b3d575bf6b 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -304,6 +304,16 @@ - name: Check reread results debug: var=reread_result.stdout_lines + - name: Take ownership of /etc/grafana + file: + path: /etc/grafana + owner: pmm + group: pmm + recurse: yes + + - name: Check ownership of /etc/grafana + command: ls -la /etc/grafana + - name: Restart services EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' and is_docker command: supervisorctl {{ item.1 }} {{ item.0 }} From 4749491beafb6ff25908d71d16b8ff54e233123d Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Nov 2023 19:08:37 +0000 Subject: [PATCH 11/35] PMM-12693 try other options for /etc/grafana --- update/ansible/playbook/tasks/update.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index b3d575bf6b..79cbc635e8 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -308,7 +308,8 @@ file: path: /etc/grafana owner: pmm - group: pmm + group: grafana + mode: 0644 recurse: yes - name: Check ownership of /etc/grafana @@ -331,7 +332,7 @@ shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log - name: Check grafana logs - shell: tail -n 200 /srv/logs/grafana.log + shell: cat /srv/logs/grafana.log - name: Delete unused grafana artifacts file: From 57ac5ee38b0c46d1439d692185be0dac0c850db3 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 28 Nov 2023 08:54:21 +0000 Subject: [PATCH 12/35] PMM-12693 try different permissions on /etc/grafana --- .../playbook/tasks/roles/grafana/tasks/main.yml | 13 ++++--------- update/ansible/playbook/tasks/update.yml | 17 ++++++----------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml index 2d92a6e3cf..108e95376a 100644 --- a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml @@ -5,23 +5,18 @@ state: directory owner: pmm group: pmm - mode: 0775 + mode: 0755 loop: - /srv/grafana - /srv/grafana/plugins -- name: Set Grafana folder for plugins on /srv partition for all users - lineinfile: - path: /etc/bashrc - line: "export GF_PLUGIN_DIR=/srv/grafana/plugins" - - name: Set ownership on /etc/grafana file: path: /etc/grafana state: directory owner: pmm group: pmm - mode: 0644 + mode: 0744 recurse: true - name: Copy new version of grafana.ini @@ -30,7 +25,7 @@ dest: /etc/grafana/grafana.ini owner: pmm group: pmm - mode: 0644 + # mode: 0744 - name: Create provisioning directory file: @@ -54,6 +49,6 @@ - plugins - dashboards -- name: Upgrade grafana database (Get the latest schema) +- name: Upgrade grafana database (apply the latest schema) command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords changed_when: True diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 79cbc635e8..7b65dc31b8 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -308,15 +308,18 @@ file: path: /etc/grafana owner: pmm - group: grafana - mode: 0644 + group: pmm + mode: 0755 recurse: yes - name: Check ownership of /etc/grafana command: ls -la /etc/grafana - name: Restart services EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' and is_docker + when: + - is_docker + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: supervisorctl {{ item.1 }} {{ item.0 }} become: true changed_when: true @@ -411,12 +414,6 @@ # SIGUSR2 is sent to supervisord by pmm-managed right before the update for logging to work correctly. # We use that fact to show what was restarted during the update. - - name: Get supervisord logs EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - shell: supervisorctl maintail -100000 | tac | awk '!flag; /received SIGUSR2/{flag = 1};' | tac - register: maintail_result - changed_when: False - - name: Get supervisord logs EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' shell: supervisorctl maintail -100000 | tac | awk '!flag; /received SIGUSR2/{flag = 1};' | tac @@ -435,5 +432,3 @@ file: state: absent path: /var/cache/yum - - From 02573be79cee86c837e4a0f1c695d7bbc73e25f5 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 28 Nov 2023 17:10:42 +0000 Subject: [PATCH 13/35] PMM-12693 remove grafana user --- build/ansible/roles/pmm-images/tasks/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 4323111ca8..2b855f8463 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -101,7 +101,6 @@ loop: - { name: pmm, gid: 1000 } - { name: nginx, gid: 999 } - - { name: grafana, gid: 1002 } - { name: clickhouse, gid: 997 } - { name: pmm-agent, gid: 996 } @@ -117,7 +116,6 @@ loop: - { name: pmm, uid: 1000, comment: "PMM Server", shell: "/bin/false", home: "/home/pmm", group: pmm, } - { name: nginx, uid: 999, comment: "nginx user", shell: "/sbin/nologin", home: "/var/cache/nginx", group: nginx, } - - { name: grafana, uid: 1002, comment: "Grafana Server", shell: "/sbin/nologin", home: "/home/grafana", group: grafana, } - { name: clickhouse, uid: 997, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/var/lib/clickhouse", group: clickhouse, } - { name: pmm-agent, uid: 996, comment: "pmm-agent", shell: "/bin/false", home: "/usr/local/percona/", group: pmm-agent, } when: ansible_virtualization_type == "docker" From 38049d512597c0721a2c532d47eb7fd5224ccc0b Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 28 Nov 2023 22:17:42 +0000 Subject: [PATCH 14/35] PMM-12693 remove grafana user --- build/docker/server/create_users.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build/docker/server/create_users.sh b/build/docker/server/create_users.sh index 7cf342d344..78c1e00f10 100644 --- a/build/docker/server/create_users.sh +++ b/build/docker/server/create_users.sh @@ -3,7 +3,6 @@ users=( "pmm:1000:/bin/false:/home/pmm:pmm" "nginx:999:/sbin/nologin:/var/cache/nginx:nginx" - "grafana:998:/sbin/nologin:/etc/grafana:grafana" "clickhouse:997:/sbin/nologin:/var/lib/clickhouse:clickhouse" "pmm-agent:996:/bin/false:/usr/local/percona/:pmm-agent" ) From 0a7bd7d5b714900b1a23780da7b62a51ab00f257 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 28 Nov 2023 22:36:02 +0000 Subject: [PATCH 15/35] PMM-12693 set dir permissions for grafana-dashboards --- build/ansible/roles/supervisord-init/tasks/main.yml | 6 +++--- build/packages/rpm/server/SPECS/percona-dashboards.spec | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/build/ansible/roles/supervisord-init/tasks/main.yml b/build/ansible/roles/supervisord-init/tasks/main.yml index 9dd37bcb05..b2ed329987 100644 --- a/build/ansible/roles/supervisord-init/tasks/main.yml +++ b/build/ansible/roles/supervisord-init/tasks/main.yml @@ -20,7 +20,7 @@ - name: Configure supervisor EL9 | Create a default configuration file for supervisord when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' shell: /usr/local/bin/echo_supervisord_conf > /etc/supervisord.conf - ignore_errors: yes + ignore_errors: True - name: Configure supervisor | Modify supervisord.conf ini_file: @@ -66,7 +66,7 @@ - name: Configure supervisor | Create /etc/supervisord.d dir file: path: /etc/supervisord.d - mode: "0755" + mode: 0755 state: directory - name: Configure supervisor | Add /etc/tmpfiles.d/supervisor.conf config @@ -112,4 +112,4 @@ - name: Debug | Print the contents of supervisord.conf debug: msg: - - "{{ lookup('file', '/etc/supervisord.conf') }}" + - "{{ lookup('file', '/etc/supervisord.conf') | split('\n') }}" diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec index d5b756cf24..a2b840df70 100644 --- a/build/packages/rpm/server/SPECS/percona-dashboards.spec +++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec @@ -7,13 +7,13 @@ %global commit ad4af6808bcd361284e8eb8cd1f36b1e98e32bce %global shortcommit %(c=%{commit}; echo ${c:0:7}) %define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 20 +%define release 21 %define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} Name: percona-dashboards Version: %{version} Release: %{rpm_release} -Summary: Grafana dashboards for MySQL and MongoDB monitoring using Prometheus +Summary: Grafana dashboards for MySQL, PostgreSQL and MongoDB monitoring License: AGPLv3 URL: https://%{provider} @@ -53,10 +53,13 @@ echo %{version} > %{buildroot}%{_datadir}/%{name}/VERSION %files %license LICENSE %doc README.md LICENSE -%attr(-,grafana,grafana) %{_datadir}/%{name} +%attr(-,pmm,pmm) %{_datadir}/%{name} %changelog +* Wed Nov 29 2023 Alex Demidoff - 3.0.0-21 +- PMM-12693 Run Grafana as non-root user + * Wed Jul 12 2023 Alex Tymchuk - 2.39.0-20 - PMM-12231 Set grafana user as owner of plugins directory From e1bc5dd3fcbf2a16b32c9a14d9f4b09a3951dfc9 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 28 Nov 2023 22:36:28 +0000 Subject: [PATCH 16/35] PMM-12693 do not install packages twice --- update/ansible/playbook/tasks/update.yml | 91 +++++++----------------- 1 file changed, 26 insertions(+), 65 deletions(-) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 7b65dc31b8..1e8144c873 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -8,33 +8,24 @@ environment: PATH: /usr/local/bin:{{ ansible_env.PATH }} - vars: - pmm_packages: - - percona-victoriametrics - - percona-qan-api2 - - percona-alertmanager - - pmm-managed - - pmm-update - - pmm-client - - pmm-dump - - vmproxy pre_tasks: - name: detect /srv/pmm-distribution stat: path: /srv/pmm-distribution - no_log: yes + no_log: true register: srv_pmm_distribution - name: detect containers set_fact: is_docker: '{{ lookup("file", "/srv/pmm-distribution") == "docker" }}' - no_log: yes + no_log: true when: srv_pmm_distribution.stat.exists - name: force container set_fact: - is_docker: True + is_docker: true when: is_docker is undefined + tasks: - name: Enable maintenance mode copy: @@ -42,53 +33,12 @@ dest: /usr/share/pmm-server/maintenance/ mode: 0644 - - name: Remove percona-dashboard without architecture - yum: - name: percona-dashboards.*noarch - state: absent - - # see https://jira.percona.com/browse/PMM-8492 for details about a issue - - name: Delete experimental repo file in 2.16 version - file: - path: "/etc/yum.repos.d/percona-original-experimental.repo" - state: absent - register: experimental_repo_existed - - - name: Update percona-dashboards package - yum: - name: - - percona-dashboards - - percona-grafana - state: latest - - - name: Cleanup yum metadata - command: yum clean metadata - register: yum_clean_result - changed_when: "'Cleaning repos' in yum_clean_result.stdout" - when: experimental_repo_existed.changed - tags: - - skip_ansible_lint # '503 Tasks that run when changed should likely be handlers'. - # The handler looks bad in this case - - # TODO: join with the command above - name: Cleanup yum metadata command: yum clean metadata become: true tags: - skip_ansible_lint - # Split download and update to produce a bit more of progress output. - - name: Download pmm packages - yum: - name: "{{ pmm_packages }}" - state: latest - download_only: yes - - - name: Update pmm packages - yum: - name: "{{ pmm_packages }}" - state: latest - - name: Create supervisord dir file: path: /etc/supervisord.d/ @@ -170,7 +120,7 @@ - not is_supervisor_running.stat.exists - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - ansible_distribution_major_version == '9' - shell: /usr/local/bin/supervisord -c /etc/supervisord.conf & + shell: supervisord -c /etc/supervisord.conf & - name: Wait until postgres port is present before continuing wait_for: @@ -200,9 +150,10 @@ # See https://github.com/Supervisor/supervisor/issues/1264 for explanation # why we do reread + stop/remove/add instead of using supervisorctl Ansible module. - - name: Reread supervisord configuration EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: supervisorctl reread register: reread_result changed_when: "'No config updates to processes' not in reread_result.stdout" @@ -231,7 +182,7 @@ yum: name: ansible state: absent - ignore_errors: yes + ignore_errors: true - name: Install ansible-core RPM | EL9 when: @@ -296,7 +247,9 @@ replace: "" - name: Reread supervisord configuration again EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: supervisorctl reread register: reread_result changed_when: "'No config updates to processes' not in reread_result.stdout" @@ -309,8 +262,8 @@ path: /etc/grafana owner: pmm group: pmm - mode: 0755 - recurse: yes + mode: 0744 + recurse: true - name: Check ownership of /etc/grafana command: ls -la /etc/grafana @@ -380,21 +333,27 @@ changed_when: True - name: Reread pmm-update-perform-init supervisor config EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: supervisorctl reread register: reread_init__result changed_when: "'No config updates to processes' not in reread_init__result.stdout" # restarting pmm-managed to regenerate /etc/alertmanager.yml - name: Restart pmm-managed EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: supervisorctl {{ item }} pmm-managed become: true changed_when: true with_items: ["stop", "remove", "add"] - name: Update/restart other services EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: supervisorctl update register: update_result changed_when: "'updated' in update_result.stdout" @@ -415,7 +374,9 @@ # SIGUSR2 is sent to supervisord by pmm-managed right before the update for logging to work correctly. # We use that fact to show what was restarted during the update. - name: Get supervisord logs EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' shell: supervisorctl maintail -100000 | tac | awk '!flag; /received SIGUSR2/{flag = 1};' | tac register: maintail_result changed_when: False From 8b3179b9a620f5fdef655a77e8f0f94acfb44861 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Wed, 29 Nov 2023 01:01:38 +0000 Subject: [PATCH 17/35] PMM-12693 remove CentOS 7 tasks --- build/ansible/pmm/post-build-actions.yml | 17 +------- build/ansible/roles/pmm-images/tasks/main.yml | 39 +++---------------- .../tasks/roles/initialization/tasks/main.yml | 11 ------ .../playbook/tasks/roles/nginx/tasks/main.yml | 20 ---------- 4 files changed, 7 insertions(+), 80 deletions(-) diff --git a/build/ansible/pmm/post-build-actions.yml b/build/ansible/pmm/post-build-actions.yml index 6e4b431400..c6986c4eb5 100644 --- a/build/ansible/pmm/post-build-actions.yml +++ b/build/ansible/pmm/post-build-actions.yml @@ -70,12 +70,6 @@ --server-address=127.0.0.1:443 --server-insecure-tls - - name: Reread supervisord configuration EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl reread - register: reread_result - changed_when: "'No config updates to processes' not in reread_result.stdout" - - name: Reread supervisord configuration EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' command: /usr/local/bin/supervisorctl reread @@ -85,13 +79,6 @@ - name: See what services are running debug: var=reread_result.stdout_lines - - name: Stop pmm-managed before deleting the database EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - supervisorctl: - name: pmm-managed - state: stopped - ignore_errors: True - - name: Stop pmm-managed before deleting the database EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' supervisorctl: @@ -206,7 +193,7 @@ path: /srv/victoriametrics owner: pmm group: pmm - mode: '0775' + mode: 0775 with_items: - absent - directory @@ -217,4 +204,4 @@ state: directory owner: pmm group: pmm - mode: "0775" + mode: 0775 diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 2b855f8463..769a344be6 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -4,21 +4,10 @@ state: present key: https://downloads.percona.com/downloads/RPM-GPG-KEY-percona -- name: Packages | Add PMM3 Server YUM repository for EL7 - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - yum_repository: - name: pmm-server - description: PMM Server YUM repository - x86_64 - baseurl: https://repo.percona.com/pmm3-components/yum/{{ pmm_server_repo }}/7/RPMS/x86_64/ - gpgcheck: yes - enabled: yes - gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY - - name: Packages | Add PMM3 Server YUM repository for EL9 when: - - (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' yum_repository: name: pmm-server description: PMM Server YUM repository - x86_64 @@ -27,7 +16,7 @@ enabled: yes gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY -# local yum repo for building of pmm server docker image in autobuild jobs +# local yum repo for building pmm server docker image in autobuild jobs - name: PMM | Add local YUM repository when: ansible_virtualization_type == "docker" yum_repository: @@ -47,16 +36,6 @@ state: installed ignore_errors: True -- name: Packages | Update OS EL7 - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - yum: - name: "*" - state: latest - exclude: "ansible*" - disablerepo: percona-release-x86_64 - - name: Packages | Update OS EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -66,18 +45,10 @@ state: latest disablerepo: percona-release-x86_64 -- name: Packages | Install OS tools for EL7 - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - yum: - name: - - python2-pip - - rsync - - name: Packages | Install OS tools for EL9 when: - - (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' yum: name: - python3-pip diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 33a42cdcd5..0ddb2227e6 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -53,17 +53,6 @@ mode: 0644 when: docker_upgrade -# PMM-10858 - In certain environments, including AWS EC2, some of the -# EPEL repository mirrors do not respond within the time limit defined -# by pmm-update which is currently set to 30 seconds. This was causing -# supervisord to kill pmm-update-checker -- name: Update repository settings - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - command: yum-config-manager --setopt=epel.timeout=1 --save - changed_when: True - - name: Clean yum metadata command: yum clean metadata become: true diff --git a/update/ansible/playbook/tasks/roles/nginx/tasks/main.yml b/update/ansible/playbook/tasks/roles/nginx/tasks/main.yml index 57d7f95ae6..1ae7e56102 100644 --- a/update/ansible/playbook/tasks/roles/nginx/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/nginx/tasks/main.yml @@ -1,16 +1,5 @@ --- # We already have nginx package in epel repo -- name: Add Nginx repository for RHEL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - yum_repository: - name: nginx - description: nginx repo - baseurl: http://nginx.org/packages/centos/7/$basearch/ - gpgcheck: no - enabled: no - - name: Add Nginx repository for RHEL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -81,12 +70,3 @@ src: local-rss.xml dest: /usr/share/pmm-server/static/ mode: 0644 - -# - name: Restart nginx -# command: /usr/local/bin/supervisorctl {{ item }} nginx -# become: true -# changed_when: True -# loop: -# - "stop" -# - "remove" -# - "add" From 8ff568a6368191f458fcb13e274a027d5747d9d2 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Wed, 29 Nov 2023 01:02:07 +0000 Subject: [PATCH 18/35] PMM-12693 move grafana bootstrap to an earlier time --- update/ansible/playbook/tasks/update.yml | 31 +++++++++--------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 1e8144c873..d34efa784d 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -81,6 +81,14 @@ include_role: name: postgres + - name: Install nginx + include_role: + name: nginx + + - name: Install clickhouse + include_role: + name: clickhouse + # Set forking type to 'simple' - name: Configure systemd when: not is_docker @@ -126,6 +134,7 @@ wait_for: host: localhost port: 5432 + timeout: 150 - name: Run initialization playbook include_role: @@ -207,14 +216,6 @@ exclude: - nginx* - - name: Install nginx - include_role: - name: nginx - - - name: Install clickhouse - include_role: - name: clickhouse - # Fix things that should be fixed before restarts. - name: Stop systemd pmm-agent service, if running @@ -257,17 +258,6 @@ - name: Check reread results debug: var=reread_result.stdout_lines - - name: Take ownership of /etc/grafana - file: - path: /etc/grafana - owner: pmm - group: pmm - mode: 0744 - recurse: true - - - name: Check ownership of /etc/grafana - command: ls -la /etc/grafana - - name: Restart services EL9 when: - is_docker @@ -287,6 +277,9 @@ - name: Check supervisord logs shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log + - name: Check ownership of /etc/grafana + command: ls -la /etc/grafana + - name: Check grafana logs shell: cat /srv/logs/grafana.log From 37a6510e73d0b8c1f339e20a495d94d86a584c2d Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Wed, 29 Nov 2023 01:28:14 +0000 Subject: [PATCH 19/35] PMM-12693 remove more CentOS 7 tasks --- build/ansible/pmm/post-build-actions.yml | 33 ++++----- build/ansible/roles/ami-ovf/tasks/main.yml | 35 --------- build/ansible/roles/cloud-node/tasks/main.yml | 38 ---------- .../roles/supervisord-init/tasks/main.yml | 9 --- .../tasks/roles/clickhouse/tasks/main.yml | 15 ---- .../roles/dashboards_upgrade/tasks/main.yml | 10 --- .../tasks/roles/postgres/tasks/main.yml | 73 ------------------- update/ansible/playbook/tasks/update.yml | 5 -- 8 files changed, 13 insertions(+), 205 deletions(-) diff --git a/build/ansible/pmm/post-build-actions.yml b/build/ansible/pmm/post-build-actions.yml index c6986c4eb5..07104b2363 100644 --- a/build/ansible/pmm/post-build-actions.yml +++ b/build/ansible/pmm/post-build-actions.yml @@ -71,7 +71,9 @@ --server-insecure-tls - name: Reread supervisord configuration EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' command: /usr/local/bin/supervisorctl reread register: reread_result changed_when: "'No config updates to processes' not in reread_result.stdout" @@ -80,13 +82,13 @@ debug: var=reread_result.stdout_lines - name: Stop pmm-managed before deleting the database EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' supervisorctl: name: pmm-managed state: stopped supervisorctl_path: /usr/local/bin/supervisorctl - # become: true - # ignore_errors: True - name: Supervisord stop | Stop supervisord service for AMI/OVF when: ansible_virtualization_type != "docker" @@ -114,16 +116,10 @@ ignore_errors: yes when: ansible_virtualization_type != "docker" - - name: Remove pmm-managed database EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - postgresql_db: - login_user: postgres - name: pmm-managed - state: absent - register: db_check_result - - name: Remove pmm-managed database EL9 - when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' postgresql_db: login_user: postgres name: pmm-managed @@ -136,7 +132,6 @@ msg: "pmm-managed database was removed" when: db_check_result.changed == True - - name: Remove pmm-managed role from postgres postgresql_user: name: pmm-managed @@ -153,10 +148,6 @@ when: ansible_virtualization_type != "docker" service: name=supervisord state=stopped enabled=yes - - name: Supervisord stop EL7 | Stop supervisord service for docker - when: ansible_virtualization_type == "docker" and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - shell: supervisorctl shutdown - - name: Supervisord stop EL9 | Stop supervisord service for docker when: ansible_virtualization_type == "docker" and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' shell: /usr/local/bin/supervisorctl shutdown @@ -166,8 +157,10 @@ # "yum clean all" function will only remove cache from configured yum repositories # Details: https://bugzilla.redhat.com/show_bug.cgi?id=1357083 - - name: Remove yum cache | Remove yum cache dir - command: rm -rf /var/cache/yum + - name: Cleanup yum cache + file: + state: absent + path: /var/cache/yum - name: Post-build cleanup | Cleanup build logs and data file: path={{ item }} state=absent diff --git a/build/ansible/roles/ami-ovf/tasks/main.yml b/build/ansible/roles/ami-ovf/tasks/main.yml index 9aebe59308..d9431f4631 100644 --- a/build/ansible/roles/ami-ovf/tasks/main.yml +++ b/build/ansible/roles/ami-ovf/tasks/main.yml @@ -7,18 +7,6 @@ - name: Packages | Clean up yum metadata command: yum clean metadata -- name: Packages | Add PMM3 Server release repository for EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - yum_repository: - name: pmm-server - description: PMM Server YUM repository - x86_64 - baseurl: https://repo.percona.com/pmm3-components/yum/experimental/7/RPMS/x86_64/ - gpgcheck: yes - enabled: yes - gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY - - name: Packages | Add PMM3 Server release repository for EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -31,14 +19,6 @@ enabled: yes gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY -- name: Disable SELinux | EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - selinux: - policy: targeted - state: permissive - - name: Disable SELinux | EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -49,15 +29,6 @@ policy: targeted state: permissive -- name: Add firewalld rule | EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - firewalld: port={{ item }} permanent=true state=enabled immediate=yes - with_items: - - 80/tcp - - 443/tcp - - name: Add firewalld rule | EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -86,12 +57,6 @@ - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - ansible_distribution_major_version == '9' -- name: PMM | Delete centos EL7 - shell: cd /tmp; nohup sh -c "trap 'userdel -r centos' EXIT; sleep 600" /dev/null 2>&1 & - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - - name: PMM | Delete ec2-user EL9 shell: cd /tmp; nohup sh -c "trap 'userdel -r ec2-user' EXIT; sleep 600" /dev/null 2>&1 & when: diff --git a/build/ansible/roles/cloud-node/tasks/main.yml b/build/ansible/roles/cloud-node/tasks/main.yml index b5fb6bcf6b..0f37c1b556 100644 --- a/build/ansible/roles/cloud-node/tasks/main.yml +++ b/build/ansible/roles/cloud-node/tasks/main.yml @@ -1,13 +1,5 @@ --- # Common things for all cloud images -- name: Packages | Add EPEL repository for EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - yum: - name: https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm - state: installed - - name: Packages | Add EPEL repository for EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -16,19 +8,6 @@ name: epel-release state: installed -- name: Packages | Install OS tools for EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - yum: - name: - - screen - - yum-utils - - cloud-init - - firewalld - - python2-pip - - ansible - - name: Packages | Install OS tools for EL9 when: - (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' @@ -41,16 +20,6 @@ - python3-libselinux - python3-firewall -- name: Firewalld | Start EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - - ansible_os_family == 'RedHat' - service: - name: firewalld - state: started - enabled: yes - - name: Firewalld | Start EL9 when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' @@ -137,13 +106,6 @@ dest: /etc/sudoers.d/90-admin-user mode: 0440 -- name: change cloud user EL7 | Change cloud user - when: create_admin == "true" and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - replace: - dest: /etc/cloud/cloud.cfg - regexp: "name: centos" - replace: "name: admin" - - name: change cloud user for OVF EL9 | Change cloud user when: - create_admin == "true" diff --git a/build/ansible/roles/supervisord-init/tasks/main.yml b/build/ansible/roles/supervisord-init/tasks/main.yml index b2ed329987..73df7c6da8 100644 --- a/build/ansible/roles/supervisord-init/tasks/main.yml +++ b/build/ansible/roles/supervisord-init/tasks/main.yml @@ -1,9 +1,4 @@ --- -- name: Install supervisor | EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - pip: - name: supervisor==3.4.0 - - name: Install supervisor | EL9 when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' pip: @@ -13,10 +8,6 @@ when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' shell: if [ ! -e /usr/bin/supervisord ]; then ln -s /usr/local/bin/supervisord /usr/bin/supervisord; fi -- name: Configure supervisor EL7 | Create a default configuration file for supervisord - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - shell: echo_supervisord_conf > /etc/supervisord.conf - - name: Configure supervisor EL9 | Create a default configuration file for supervisord when: (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' shell: /usr/local/bin/echo_supervisord_conf > /etc/supervisord.conf diff --git a/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml b/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml index d5fa9f845e..cdd4ddeeea 100644 --- a/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml @@ -1,14 +1,4 @@ --- -- name: Stop and remove clickhouse before update | EL7 - when: - - ansible_distribution == 'CentOS' - - ansible_distribution_major_version == '7' - command: supervisorctl {{ item }} clickhouse - changed_when: True - loop: - - stop - - remove - - name: Find supervisord's socket stat: path: /var/run/supervisor/supervisor.sock @@ -123,8 +113,3 @@ loop: - present - started - -- name: Start clickhouse EL7 - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl add clickhouse - changed_when: True diff --git a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml index 8db53ed578..2a36bbbe0e 100644 --- a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml @@ -62,16 +62,6 @@ - name: Remove the old clickhouse plugin shell: grafana cli --pluginsDir /srv/grafana/plugins plugins remove vertamedia-clickhouse-datasource || true -- name: Restart grafana with new plugins EL7 - supervisorctl: - name: grafana - state: restarted - become: true - ignore_errors: true - when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - # TODO: fix the race condition. - # We generate grafana supervisor config in pmm-managed and it may not exist at this stage - - name: Restart grafana with new plugins EL9 supervisorctl: name: grafana diff --git a/update/ansible/playbook/tasks/roles/postgres/tasks/main.yml b/update/ansible/playbook/tasks/roles/postgres/tasks/main.yml index e70c17ca80..1b1dd56d42 100644 --- a/update/ansible/playbook/tasks/roles/postgres/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/postgres/tasks/main.yml @@ -1,43 +1,5 @@ --- # Install Postgres -- name: Install Postgres for EL7 - block: - - name: Add PostgreSQL 14 YUM repository for EL7 - yum_repository: - name: percona-ppg-14 - description: PostgreSQL YUM repository - x86_64 - baseurl: http://repo.percona.com/ppg-14/yum/release/7/RPMS/x86_64 - gpgcheck: yes - enabled: yes - gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY - - - name: Add PostgreSQL 11 YUM repository for EL7 - yum_repository: - name: percona-ppg-11 - description: PostgreSQL YUM repository - x86_64 - baseurl: http://repo.percona.com/ppg-11/yum/release/7/RPMS/x86_64 - gpgcheck: yes - enabled: yes - gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY - - # we need the old postgres binary for the upgrade process - - name: Install Postgres - when: - - not ansible_check_mode - yum: - name: - - percona-postgresql14-server - - percona-postgresql14-contrib - - percona-postgresql14 - - percona-postgresql11-server - - percona-postgresql11-contrib - - percona-postgresql11 - - python-psycopg2 # Python PostgreSQL database adapter§ - state: installed - when: - - ansible_distribution == "CentOS" - - ansible_distribution_major_version == "7" - - name: Install Postgres for EL9 block: - name: Add PostgreSQL 14 YUM repository for EL9 @@ -118,18 +80,6 @@ path: /var/run/supervisor/supervisor.sock register: is_upgrade - - name: Stop pmm-managed and postgres before backup database | EL7 - supervisorctl: - name: "{{ item }}" - state: stopped - loop: - - pmm-managed - - postgresql - become: true - when: - - is_upgrade.stat.exists - - ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - - name: Stop pmm-managed and postgres before backup database | EL9 supervisorctl: name: "{{ item }}" @@ -217,25 +167,11 @@ state: absent when: is_upgrade.stat.exists - - name: Reread supervisord configuration EL7 - when: is_upgrade.stat.exists and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - command: supervisorctl reread - - name: Reread supervisord configuration EL9 when: is_upgrade.stat.exists and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' command: /usr/local/bin/supervisorctl reread become: true - - name: Restart Postgres | EL7 - command: supervisorctl {{ item }} postgresql - changed_when: True - become: true - loop: - - stop - - remove - - add - when: is_upgrade.stat.exists and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - - name: Restart Postgres | EL9 command: /usr/local/bin/supervisorctl {{ item }} postgresql changed_when: True @@ -246,15 +182,6 @@ - add when: is_upgrade.stat.exists and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' - - name: Run pmm-managed again | EL7 - supervisorctl: - name: pmm-managed - state: started - become: true - when: - - is_upgrade.stat.exists - - ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' - - name: Run pmm-managed again | EL9 supervisorctl: name: pmm-managed diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index d34efa784d..c84b100af6 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -381,8 +381,3 @@ file: state: absent path: /usr/share/pmm-server/maintenance/maintenance.html - - - name: Cleanup yum cache - file: - state: absent - path: /var/cache/yum From dd3ffe6da65c976c1143c5d03dc887e04d1e906d Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Wed, 29 Nov 2023 07:45:56 +0000 Subject: [PATCH 20/35] PMM-12693 move clickhouse install back --- build/ansible/roles/pmm-images/tasks/main.yml | 32 ++++++++++++++ .../tasks/roles/clickhouse/tasks/main.yml | 2 +- update/ansible/playbook/tasks/update.yml | 44 +++---------------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index 769a344be6..f07ac17571 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -144,6 +144,38 @@ state: installed enablerepo: "{{ pmm_client_repo_name }}" +- name: Remove ansible RPM if installed | EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + yum: + name: ansible + state: absent + ignore_errors: true + +- name: Install ansible-core RPM | EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + yum: + name: + - ansible-core + - ansible-collection-community-general + - ansible-collection-community-postgresql + - ansible-collection-ansible-posix + state: present + +- name: Update system packages EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + - not ansible_check_mode + yum: + name: "*" + state: latest + exclude: + - nginx* + - name: Disable pmm-agent service | Disable pmm-agent when: ansible_virtualization_type != "docker" service: name=pmm-agent state=stopped enabled=no diff --git a/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml b/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml index cdd4ddeeea..40607368e0 100644 --- a/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/clickhouse/tasks/main.yml @@ -97,7 +97,7 @@ path: "/usr/bin/clickhouse-odbc-bridge" state: absent -- name: Change ownership for clickhouse directory +- name: Change ownership of clickhouse directory file: path: /srv/clickhouse/ owner: root diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index c84b100af6..099a75ac49 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -81,14 +81,6 @@ include_role: name: postgres - - name: Install nginx - include_role: - name: nginx - - - name: Install clickhouse - include_role: - name: clickhouse - # Set forking type to 'simple' - name: Configure systemd when: not is_docker @@ -184,37 +176,13 @@ - name: Wait for pmm-managed pause: seconds=10 - - name: Remove ansible RPM if installed | EL9 - when: - - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - - ansible_distribution_major_version == '9' - yum: - name: ansible - state: absent - ignore_errors: true - - - name: Install ansible-core RPM | EL9 - when: - - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - - ansible_distribution_major_version == '9' - yum: - name: - - ansible-core - - ansible-collection-community-general - - ansible-collection-community-postgresql - - ansible-collection-ansible-posix - state: present + - name: Install nginx + include_role: + name: nginx - - name: Update system packages EL9 - when: - - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - - ansible_distribution_major_version == '9' - - not ansible_check_mode - yum: - name: "*" - state: latest - exclude: - - nginx* + - name: Install clickhouse + include_role: + name: clickhouse # Fix things that should be fixed before restarts. From 220202563e7396d3cac6fd7a76f87f5c1f4711e5 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Wed, 29 Nov 2023 09:13:10 +0000 Subject: [PATCH 21/35] PMM-12693 do not upgrade sqlite --- build/ansible/roles/pmm-images/tasks/main.yml | 11 ----------- .../playbook/tasks/roles/grafana/tasks/main.yml | 8 ++++---- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml index f07ac17571..e2d205edb9 100644 --- a/build/ansible/roles/pmm-images/tasks/main.yml +++ b/build/ansible/roles/pmm-images/tasks/main.yml @@ -165,17 +165,6 @@ - ansible-collection-ansible-posix state: present -- name: Update system packages EL9 - when: - - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - - ansible_distribution_major_version == '9' - - not ansible_check_mode - yum: - name: "*" - state: latest - exclude: - - nginx* - - name: Disable pmm-agent service | Disable pmm-agent when: ansible_virtualization_type != "docker" service: name=pmm-agent state=stopped enabled=no diff --git a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml index 108e95376a..cd007335da 100644 --- a/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/grafana/tasks/main.yml @@ -25,7 +25,6 @@ dest: /etc/grafana/grafana.ini owner: pmm group: pmm - # mode: 0744 - name: Create provisioning directory file: @@ -49,6 +48,7 @@ - plugins - dashboards -- name: Upgrade grafana database (apply the latest schema) - command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords - changed_when: True +# This was redundant, as the schema is migrated during the startup phase +# - name: Upgrade grafana database (apply the latest schema) +# command: grafana cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin data-migration encrypt-datasource-passwords +# changed_when: True From db324376d78c7a86a276f4fa993493dcc4a71f24 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Wed, 29 Nov 2023 23:20:52 +0000 Subject: [PATCH 22/35] PMM-12693 clean up the dashboards playbook --- .../roles/dashboards_upgrade/tasks/main.yml | 39 +------------------ 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml index 2a36bbbe0e..87af436cc2 100644 --- a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml @@ -1,37 +1,4 @@ --- -- name: Check that old plugin dir exists - stat: - path: /var/lib/grafana/plugins/ - register: old_plugin_dir_exist - -- name: Copy plugins from old plugin directory - block: - - name: Find custom plugins in old plugin dir - find: - paths: /var/lib/grafana/plugins - recurse: no - file_type: directory - excludes: - - "*-???????" - - pmm-app - register: custom_plugins - - - name: Synchronization plugins (for versions before 2.22.0) - synchronize: - src: "{{ item['path'] }}" - dest: "/srv/grafana/plugins/" - loop: "{{ custom_plugins['files'] }}" - - - name: Find custom plugin in old plugin dir - find: - paths: /var/lib/grafana/plugins - recurse: no - file_type: directory - excludes: - - "*-???????" - - pmm-app - when: old_plugin_dir_exist.stat.exists - - name: Get plugin list register: plugin_list find: @@ -39,7 +6,7 @@ depth: 2 file_type: directory -- name: Delete existing dist folder +- name: Delete redundant dist folders file: path: "/srv/grafana/plugins/{{ item['path'].split('/')[-1] }}" state: absent @@ -59,14 +26,10 @@ mode: 0775 recurse: yes -- name: Remove the old clickhouse plugin - shell: grafana cli --pluginsDir /srv/grafana/plugins plugins remove vertamedia-clickhouse-datasource || true - - name: Restart grafana with new plugins EL9 supervisorctl: name: grafana state: restarted - supervisorctl_path: /usr/local/bin/supervisorctl become: true when: - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' From 8d41a4510f564c4fd16577cae9c92680c95a78c9 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 30 Nov 2023 12:06:27 +0000 Subject: [PATCH 23/35] PMM-12693 clean up the update playbook --- update/ansible/playbook/tasks/update.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index 099a75ac49..cd2bda5dc2 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -245,19 +245,9 @@ - name: Check supervisord logs shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log - - name: Check ownership of /etc/grafana - command: ls -la /etc/grafana - - name: Check grafana logs shell: cat /srv/logs/grafana.log - - name: Delete unused grafana artifacts - file: - state: absent - path: "{{ item }}" - loop: - - /usr/share/grafana/data/grafana.db - - name: Fix grafana fields type postgresql_query: db: grafana @@ -276,16 +266,11 @@ when: not ansible_check_mode # we need to put this step as one of the last steps, because it removes pmm.ini and /etc/alertmanager.yml - - name: Remove old or redundant packages + - name: Remove redundant packages yum: state: absent name: - - percona-qan-app # https://jira.percona.com/browse/PMM-6766 - - mariadb-libs # https://jira.percona.com/browse/PMM-5215 - logrotate # https://jira.percona.com/browse/PMM-7627 - - pmm-server # https://jira.percona.com/browse/PMM-11239 - - screen - - yum-cron # Regenerating pmm.ini and enabling pmm-update-perform-init - name: Generate new supervisor config From 87efdce131cbd96deb5e5da13a1a012f86a56e51 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 1 Dec 2023 00:22:34 +0300 Subject: [PATCH 24/35] Update build/packages/rpm/server/SPECS/percona-dashboards.spec Co-authored-by: Nurlan Moldomurov --- build/packages/rpm/server/SPECS/percona-dashboards.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec index a2b840df70..5a1a895ed3 100644 --- a/build/packages/rpm/server/SPECS/percona-dashboards.spec +++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec @@ -13,7 +13,7 @@ Name: percona-dashboards Version: %{version} Release: %{rpm_release} -Summary: Grafana dashboards for MySQL, PostgreSQL and MongoDB monitoring +Summary: Grafana dashboards for monitoring License: AGPLv3 URL: https://%{provider} From 1ed4b2e583d3753744204487685de606cf35bf63 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 30 Nov 2023 21:30:27 +0000 Subject: [PATCH 25/35] PMM-12693 follow up on review --- build/scripts/build-server-rpm-all | 1 - .../roles/dashboards_upgrade/tasks/main.yml | 2 +- .../tasks/roles/initialization/tasks/main.yml | 20 ------------------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/build/scripts/build-server-rpm-all b/build/scripts/build-server-rpm-all index 6d65e16600..7b21335b36 100755 --- a/build/scripts/build-server-rpm-all +++ b/build/scripts/build-server-rpm-all @@ -10,7 +10,6 @@ ${bin_dir}/build-server-rpm pmm-managed pmm ${bin_dir}/build-server-rpm percona-qan-api2 pmm ${bin_dir}/build-server-rpm pmm-update pmm ${bin_dir}/build-server-rpm pmm-dump -# ${bin_dir}/build-server-rpm grafana-db-migrator ${bin_dir}/build-server-rpm vmproxy pmm # 3rd-party diff --git a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml index 87af436cc2..a9c0a43124 100644 --- a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml @@ -44,5 +44,5 @@ dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION owner: pmm group: pmm - mode: 0444 + mode: 0666 remote_src: yes diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 0ddb2227e6..cf98fe6918 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -63,16 +63,6 @@ path: /srv/backup state: directory -- name: Check if Postgres 11 exists - stat: - path: /srv/postgres - register: is_postgres_11 - -- name: Upgrade Postgres database - include_role: - name: postgres - when: is_postgres_11.stat.exists - - name: Create grafana database in postgres postgresql_db: name: grafana @@ -97,14 +87,6 @@ - name: Create empty configuration file for VictoriaMetrics file: path=/etc/victoriametrics-promscrape.yml state=touch owner=pmm group=pmm -- name: Change default admin id - postgresql_query: - db: grafana - query: UPDATE "user" SET id='1' WHERE login='admin'; - when: - - docker_upgrade - - not ansible_check_mode - - name: Check if we need an update or not include_role: name: dashboards_upgrade @@ -126,5 +108,3 @@ path: /usr/share/pmm-server/maintenance/maintenance.html # We use current_version_file['failed'] because we don't want to run this on creating container when: docker_upgrade - - From 60679c7d657518aa27733bc2ae7a2177f79fdc22 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 1 Dec 2023 10:55:26 +0000 Subject: [PATCH 26/35] PMM-12693 move the version copying out of dashboards_update --- .../tasks/roles/dashboards_upgrade/tasks/main.yml | 9 --------- .../tasks/roles/initialization/tasks/main.yml | 11 ++++++++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml index a9c0a43124..308cb64d85 100644 --- a/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/dashboards_upgrade/tasks/main.yml @@ -37,12 +37,3 @@ ignore_errors: true # TODO: fix the race condition. # We generate grafana supervisor config in pmm-managed and it may not exist at this stage - -- name: Copy file with image version - copy: - src: /usr/share/percona-dashboards/VERSION - dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION - owner: pmm - group: pmm - mode: 0666 - remote_src: yes diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index e2d15ebf2e..cffa2f73b0 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -118,9 +118,14 @@ - name: Create empty configuration file for VictoriaMetrics file: path=/etc/victoriametrics-promscrape.yml state=touch owner=pmm group=pmm -- name: Check if we need an upgrade - include_role: - name: dashboards_upgrade +- name: Copy file with image version + copy: + src: /usr/share/percona-dashboards/VERSION + dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION + owner: pmm + group: pmm + mode: 0666 + remote_src: yes when: not pmm_current_version is version(pmm_image_version, '>=') - name: Finalization From 1971153a82d70ff131aa1806591ff9bae3b86413 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 1 Dec 2023 14:45:30 +0000 Subject: [PATCH 27/35] PMM-12693 trigger the build From e4ba6a7c499ea404ca6b3c882f910c109e9aa1dc Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 1 Dec 2023 16:33:36 +0000 Subject: [PATCH 28/35] PMM-12693 hackily start pmm-agent --- .github/workflows/managed.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 3e46bb4d39..8fc333c4c4 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -80,6 +80,10 @@ jobs: - name: Update binaries run: docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy + # TODO: fix it and make sure it runs when 3-dev-container starts up + - name: Run pmm-agent + run: docker exec -i pmm-server supervisorctl start pmm-agent + - name: Run tests run: docker exec -i pmm-server make -C managed test-cover From 8853957e4ca8e877512a706ae088f5d68e128f81 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 08:25:49 +0000 Subject: [PATCH 29/35] PMM-12693 check the status of services --- .github/workflows/managed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 8fc333c4c4..07d1c2e0ae 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -80,9 +80,9 @@ jobs: - name: Update binaries run: docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy - # TODO: fix it and make sure it runs when 3-dev-container starts up - - name: Run pmm-agent - run: docker exec -i pmm-server supervisorctl start pmm-agent + # TODO: remove this debug step + - name: Check the status of services + run: docker exec -i pmm-server supervisorctl status - name: Run tests run: docker exec -i pmm-server make -C managed test-cover From e8b4e0b8a3d491fc9ec9c523a86f34736c8514fc Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 10:03:58 +0000 Subject: [PATCH 30/35] PMM-7 wait for container to get healthy --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 94c955890f..56c874f910 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ env-update-image: ## Pull latest dev image env-compose-up: env-update-image COMPOSE_PROFILES=$(PROFILES) \ - docker compose up --detach --renew-anon-volumes --remove-orphans + docker compose up --detach --renew-anon-volumes --remove-orphans --wait --wait-timeout 100 env-devcontainer: docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server .devcontainer/setup.py From 30f95c59bd4239b19a6ca9b50d6305ee85288940 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 10:21:15 +0000 Subject: [PATCH 31/35] PMM-7 debug: output grafana logs --- .github/workflows/managed.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 07d1c2e0ae..9c4e80ac16 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -82,7 +82,9 @@ jobs: # TODO: remove this debug step - name: Check the status of services - run: docker exec -i pmm-server supervisorctl status + run: | + docker exec -t pmm-server supervisorctl status || : + docker exec -t pmm-server cat /srv/logs/grafana.log - name: Run tests run: docker exec -i pmm-server make -C managed test-cover From 087b2382d4ef544024cc9dbca0afa4068046009d Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 11:44:57 +0000 Subject: [PATCH 32/35] PMM-7 debug the ownership of /etc/grafana/grafana.ini --- .github/workflows/managed.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 9c4e80ac16..d06386d0f6 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -78,13 +78,16 @@ jobs: run: docker exec -i pmm-server git config --global --add safe.directory /root/go/src/github.com/percona/pmm - name: Update binaries - run: docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy + run: | + docker exec -t pmm-server ls -la /etc/grafana + docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy # TODO: remove this debug step - name: Check the status of services run: | docker exec -t pmm-server supervisorctl status || : docker exec -t pmm-server cat /srv/logs/grafana.log + docker exec -t pmm-server ls -la /etc/grafana - name: Run tests run: docker exec -i pmm-server make -C managed test-cover From 7e2cfaeb43bbb37706c0e38b25bedf44c49d19f0 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 12:19:44 +0000 Subject: [PATCH 33/35] PMM-7 change ownership of /etc/grafana to pmm --- .github/workflows/managed.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index d06386d0f6..16189dd75b 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -79,7 +79,9 @@ jobs: - name: Update binaries run: | - docker exec -t pmm-server ls -la /etc/grafana + # We need to make this directory owned by pmm, since it's currently owned by the grafana user in the devcontainer. + # TODO: remove the line below after this PR is merged to v3. + docker exec -t pmm-server chown -R pmm:pmm /etc/grafana docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy # TODO: remove this debug step From cdfd73191c495260a2c6e4472fc4ba95db3f7353 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 13:03:25 +0000 Subject: [PATCH 34/35] PMM-7 change ownership of /srv/grafana to pmm --- .github/workflows/managed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 16189dd75b..1bf1bd1090 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -81,7 +81,7 @@ jobs: run: | # We need to make this directory owned by pmm, since it's currently owned by the grafana user in the devcontainer. # TODO: remove the line below after this PR is merged to v3. - docker exec -t pmm-server chown -R pmm:pmm /etc/grafana + docker exec -t pmm-server chown -R pmm:pmm /etc/grafana /srv/grafana docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy # TODO: remove this debug step From 8d9ac7ea685caeb954323513e271dca1ddea5707 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 4 Dec 2023 13:38:15 +0000 Subject: [PATCH 35/35] PMM-7 cleanup the workflow --- .github/workflows/managed.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index 1bf1bd1090..3e7d2e8ea9 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -84,12 +84,9 @@ jobs: docker exec -t pmm-server chown -R pmm:pmm /etc/grafana /srv/grafana docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy - # TODO: remove this debug step - - name: Check the status of services + - name: Check the status of components run: | docker exec -t pmm-server supervisorctl status || : - docker exec -t pmm-server cat /srv/logs/grafana.log - docker exec -t pmm-server ls -la /etc/grafana - name: Run tests run: docker exec -i pmm-server make -C managed test-cover