From b2f3fe553090ce26b6d05f5a3266dbbe8c00fae6 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Thu, 5 Dec 2024 19:36:28 +0500 Subject: [PATCH] Add support for PostgreSQL 17 (#827) Added support for Postgres 17. It is now used by default. --- automation/molecule/default/converge.yml | 4 +- automation/molecule/pg_upgrade/converge.yml | 6 +- automation/roles/upgrade/README.md | 2 +- automation/roles/upgrade/tasks/initdb.yml | 56 ++++++++++--------- .../roles/upgrade/tasks/update_config.yml | 56 ++++++++++++++++++- automation/vars/main.yml | 3 +- .../db/migrations/20241205103951_2.1.0.sql | 25 +++++++++ 7 files changed, 116 insertions(+), 36 deletions(-) create mode 100644 console/db/migrations/20241205103951_2.1.0.sql diff --git a/automation/molecule/default/converge.yml b/automation/molecule/default/converge.yml index b16d28dfb..2b9d965a9 100644 --- a/automation/molecule/default/converge.yml +++ b/automation/molecule/default/converge.yml @@ -16,7 +16,7 @@ dcs_type: "{{ ['etcd', 'consul'] | random }}" # Set 'dcs_type' to either 'etcd' or 'consul' randomly consul_node_role: server # if dcs_type: "consul" consul_bootstrap_expect: true # if dcs_type: "consul" - postgresql_version: 16 # to test custom WAL dir + postgresql_version: 17 # to test custom WAL dir pgbouncer_processes: 2 # Test multiple pgbouncer processes (so_reuseport) patroni_tags: "datacenter=dc1,key1=value1" balancer_tags: "datacenter=dc1" @@ -60,7 +60,7 @@ enable_pg_stat_kcache: true enable_pg_wait_sampling: true enable_pg_partman: true - enable_citus: "{{ 'false' if ansible_distribution_version == '24.04' else 'true' }}" # TODO Ubuntu 24.04 + # enable_citus: true # TODO Postgres 17 enable_paradedb: "{{ 'false' if ansible_distribution_release == 'bullseye' else 'true' }}" # pg_search and pg_analytics (no packages for debian 11) enable_pgvectorscale: "{{ 'true' if ansible_distribution_release in ['bookworm', 'jammy', 'noble'] else 'false' }}" # only deb packages are available pgvectorscale_version: "0.5.0" # TODO (v0.5.1 does not contain packages) diff --git a/automation/molecule/pg_upgrade/converge.yml b/automation/molecule/pg_upgrade/converge.yml index c5e27b59e..610a3a963 100644 --- a/automation/molecule/pg_upgrade/converge.yml +++ b/automation/molecule/pg_upgrade/converge.yml @@ -16,7 +16,7 @@ dcs_type: "{{ ['etcd', 'consul'] | random }}" # Set 'dcs_type' to either 'etcd' or 'consul' randomly consul_node_role: server # if dcs_type: "consul" consul_bootstrap_expect: true # if dcs_type: "consul" - postgresql_version: 14 # redefine the version to install for the upgrade test + postgresql_version: 16 # redefine the version to install for the upgrade test pgbouncer_processes: 4 # Test multiple pgbouncer processes (so_reuseport) cacheable: true delegate_to: localhost @@ -43,8 +43,8 @@ - name: Set variables for PostgreSQL upgrade test ansible.builtin.set_fact: - pg_old_version: 14 - pg_new_version: 16 + pg_old_version: 16 + pg_new_version: 17 - name: Add repository GPG key ansible.builtin.command: "rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux-{{ ansible_distribution_major_version }}" diff --git a/automation/roles/upgrade/README.md b/automation/roles/upgrade/README.md index 6b68624d4..f3060c0d6 100644 --- a/automation/roles/upgrade/README.md +++ b/automation/roles/upgrade/README.md @@ -155,7 +155,7 @@ Please see the variable file vars/[upgrade.yml](../../vars/upgrade.yml) - Perform pg_dropcluster (for Debian based) - Clear the new PostgreSQL data directory - Get the current install user (rolname with oid = 10) - - Get the current encodig and data_checksums settings + - Get the current encoding and data_checksums settings - Initialize new PostgreSQL data directory - for Debain based: on all database servers to create default config files - for RedHat based: on the Primary only diff --git a/automation/roles/upgrade/tasks/initdb.yml b/automation/roles/upgrade/tasks/initdb.yml index 21ce13b0c..df92675aa 100644 --- a/automation/roles/upgrade/tasks/initdb.yml +++ b/automation/roles/upgrade/tasks/initdb.yml @@ -49,19 +49,21 @@ - name: Get the current encodig and data_checksums settings ansible.builtin.command: >- - {{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc - "show {{ item }}" + {{ pg_old_bindir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -tAXc " + select row_to_json(pg_settings) + from ( + select + (select datcollate from pg_database where datname = current_database()) as lc_collate, + (select datctype from pg_database where datname = current_database()) as lc_ctype, + current_setting('lc_messages') as lc_messages, + current_setting('lc_monetary') as lc_monetary, + current_setting('lc_numeric') as lc_numeric, + current_setting('lc_time') as lc_time, + current_setting('server_encoding') as server_encoding, + current_setting('data_checksums') as data_checksums + ) pg_settings" changed_when: false register: pg_settings - loop: - - server_encoding - - lc_collate - - lc_ctype - - lc_messages - - lc_monetary - - lc_numeric - - lc_time - - data_checksums when: - inventory_hostname in groups['primary'] @@ -72,15 +74,15 @@ /usr/bin/pg_createcluster {{ pg_new_version }} {{ postgresql_cluster_name }} --user={{ hostvars[groups['primary'][0]].pg_install_user.stdout }} --datadir={{ pg_new_datadir }} - --encoding={{ hostvars[groups['primary'][0]].pg_settings.results[0].stdout }} - --lc-collate={{ hostvars[groups['primary'][0]].pg_settings.results[1].stdout }} - --lc-ctype={{ hostvars[groups['primary'][0]].pg_settings.results[2].stdout }} - --lc-messages={{ hostvars[groups['primary'][0]].pg_settings.results[3].stdout }} - --lc-monetary={{ hostvars[groups['primary'][0]].pg_settings.results[4].stdout }} - --lc-numeric={{ hostvars[groups['primary'][0]].pg_settings.results[5].stdout }} - --lc-time={{ hostvars[groups['primary'][0]].pg_settings.results[6].stdout }} + --encoding={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).server_encoding }} + --lc-collate={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).lc_collate }} + --lc-ctype={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).lc_ctype }} + --lc-messages={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).lc_messages }} + --lc-monetary={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).lc_monetary }} + --lc-numeric={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).lc_numeric }} + --lc-time={{ (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).lc_time }} --start-conf=manual - {% if hostvars[groups['primary'][0]].pg_settings.results[7].stdout == 'on' %} + {% if (hostvars[groups['primary'][0]].pg_settings.stdout | from_json).data_checksums == 'on' %} -- --data-checksums {% endif %} when: @@ -93,14 +95,14 @@ {{ pg_new_bindir }}/initdb --username={{ pg_install_user.stdout }} --pgdata={{ pg_new_datadir }} - --encoding={{ pg_settings.results[0].stdout }} - --lc-collate={{ pg_settings.results[1].stdout }} - --lc-ctype={{ pg_settings.results[2].stdout }} - --lc-messages={{ pg_settings.results[3].stdout }} - --lc-monetary={{ pg_settings.results[4].stdout }} - --lc-numeric={{ pg_settings.results[5].stdout }} - --lc-time={{ pg_settings.results[6].stdout }} - {% if pg_settings.results[7].stdout == 'on' %} + --encoding={{ (pg_settings.stdout | from_json).server_encoding }} + --lc-collate={{ (pg_settings.stdout | from_json).lc_collate }} + --lc-ctype={{ (pg_settings.stdout | from_json).lc_ctype }} + --lc-messages={{ (pg_settings.stdout | from_json).lc_messages }} + --lc-monetary={{ (pg_settings.stdout | from_json).lc_monetary }} + --lc-numeric={{ (pg_settings.stdout | from_json).lc_numeric }} + --lc-time={{ (pg_settings.stdout | from_json).lc_time }} + {% if (pg_settings.stdout | from_json).data_checksums == 'on' %} --data-checksums {% endif %} when: diff --git a/automation/roles/upgrade/tasks/update_config.yml b/automation/roles/upgrade/tasks/update_config.yml index cfefa6517..1b5b863f6 100644 --- a/automation/roles/upgrade/tasks/update_config.yml +++ b/automation/roles/upgrade/tasks/update_config.yml @@ -213,7 +213,61 @@ when: - pg_old_version|int <= 15 and pg_new_version|int >= 16 -# TODO: Prepare the parameters for PostgreSQL 17 and etc. +- block: # db_user_namespace (removed in the PG 17) + # check if the db_user_namespace parameter is specified in the patroni.yml + - name: "Edit patroni.yml | check if the 'db_user_namespace' parameter is specified" + ansible.builtin.command: grep db_user_namespace {{ patroni_config_file }} + register: db_user_namespace_output + changed_when: false + failed_when: false + + # if defined, remove the db_user_namespace parameter from the patroni.yml + - name: "Edit patroni.yml | remove parameter: 'db_user_namespace'" + ansible.builtin.lineinfile: + path: "{{ patroni_config_file }}" + regexp: '^(\s*)db_user_namespace:.*' + state: absent + when: db_user_namespace_output.stdout | length > 0 + when: + - pg_old_version|int <= 16 and pg_new_version|int >= 17 + +- block: # old_snapshot_threshold (removed in the PG 17) + # check if the old_snapshot_threshold parameter is specified in the patroni.yml + - name: "Edit patroni.yml | check if the 'old_snapshot_threshold' parameter is specified" + ansible.builtin.command: grep old_snapshot_threshold {{ patroni_config_file }} + register: old_snapshot_threshold_output + changed_when: false + failed_when: false + + # if defined, remove the old_snapshot_threshold parameter from the patroni.yml + - name: "Edit patroni.yml | remove parameter: 'old_snapshot_threshold'" + ansible.builtin.lineinfile: + path: "{{ patroni_config_file }}" + regexp: '^(\s*)old_snapshot_threshold:.*' + state: absent + when: old_snapshot_threshold_output.stdout | length > 0 + when: + - pg_old_version|int <= 16 and pg_new_version|int >= 17 + +- block: # trace_recovery_messages (removed in the PG 17) + # check if the trace_recovery_messages parameter is specified in the patroni.yml + - name: "Edit patroni.yml | check if the 'trace_recovery_messages' parameter is specified" + ansible.builtin.command: grep trace_recovery_messages {{ patroni_config_file }} + register: trace_recovery_messages_output + changed_when: false + failed_when: false + + # if defined, remove the trace_recovery_messages parameter from the patroni.yml + - name: "Edit patroni.yml | remove parameter: 'trace_recovery_messages'" + ansible.builtin.lineinfile: + path: "{{ patroni_config_file }}" + regexp: '^(\s*)trace_recovery_messages:.*' + state: absent + when: trace_recovery_messages_output.stdout | length > 0 + when: + - pg_old_version|int <= 16 and pg_new_version|int >= 17 + +# TODO: Prepare the parameters for PostgreSQL 18 and etc. # Copy the pg_hba.conf file to a new PostgreSQL to save pg_hba rules. - name: "Copy pg_hba.conf to {{ pg_new_confdir }}" diff --git a/automation/vars/main.yml b/automation/vars/main.yml index e58b248b5..06f460cee 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -173,7 +173,7 @@ consul_services: # PostgreSQL variables -postgresql_version: 16 +postgresql_version: 17 # postgresql_data_dir: see vars/Debian.yml or vars/RedHat.yml postgresql_listen_addr: "0.0.0.0" # Listen on all interfaces. Or use "{{ inventory_hostname }},127.0.0.1" to listen on a specific IP address. postgresql_port: 5432 @@ -309,7 +309,6 @@ postgresql_parameters: - { option: "tcp_keepalives_count", value: "10" } - { option: "tcp_keepalives_idle", value: "300" } - { option: "tcp_keepalives_interval", value: "30" } -# - { option: "old_snapshot_threshold", value: "60min" } # - { option: "", value: "" } # - { option: "", value: "" } diff --git a/console/db/migrations/20241205103951_2.1.0.sql b/console/db/migrations/20241205103951_2.1.0.sql new file mode 100644 index 000000000..46079488e --- /dev/null +++ b/console/db/migrations/20241205103951_2.1.0.sql @@ -0,0 +1,25 @@ +-- +goose Up + +-- Postgres versions +INSERT INTO public.postgres_versions (major_version, release_date, end_of_life) +VALUES (17, '2024-09-26', '2029-11-08'); + +-- Extensions +UPDATE public.extensions +SET postgres_max_version = '17' +WHERE extension_name IN ( + 'pgaudit', + 'pg_cron', + 'pg_partman', + 'pg_repack', + 'pg_stat_kcache', + 'pg_wait_sampling', + 'pgvector', + 'postgis', + 'pgrouting', + 'timescaledb' +); + +-- +goose Down +DELETE FROM public.postgres_versions +WHERE major_version = 17;