From 13e8d69d68b720bf6a23df2cc36f1f7d8f5d669d Mon Sep 17 00:00:00 2001 From: Felipe Mendes Date: Tue, 5 Jul 2022 01:57:09 -0300 Subject: [PATCH] Scylla Ansible Node - Upgrade rework This commit is a rework on the steps to upgrade a cluster. In particular, the previous version didn't upgraded nodes serially, nor would abort the entire process in case of a failure. Upgrading a cluster programatically is complex in general, as we must ensure that the failure of a single node aborts the entire process. In that sense, we must also check several things such as whether the version we are upgrading to is a valid one or not, and whether we are already on 'latest' or not to prevent introducing an unecessary rolling restart to the cluster. These aspects - and many others - make testing of this extremely complex, lengthy and time consuming. However, I have ensured we at least have "minimal" coverage for some common operations. That said, the following tests have been run, whereas all tests were run concurrently in an Ubuntu 20.04/Debian 10/CentOS 7 mixed cluster: Test 1 : Install 2021.1.1 == success Test 2 : Downgrade to 2021.1.0 without specifying upgrade_version == expected failure Test 3 : Upgrade to 2021.1.2 without specifying upgrade_version == expected failure Test 4 : Upgrade to 2021.1.2 specifying upgrade_version == success Test 5 : Upgrade to 2021.1.50 specifying upgrade_version == expected failure Test 6 : Try to upgrade to 'latest' without specifying upgrade_version == expected failure Test 7 : Try to upgrade to 'latest' specifying upgrade_version == success Test 8 : Try to downgrade to 2021.1.11 with upgrade_version and NOT specifying upgrade_allow_user_manual_downgrade == expected failure Test 9 : Try to downgrade to 2021.1.11 with upgrade_version and specifying upgrade_allow_user_manual_downgrade == success Test 10 : Major upgrade to 2022.1 latest == Appears to work, can't confirm due to a bug in the release. --- ansible-scylla-node/tasks/upgrade/main.yml | 254 +++++++----- .../tasks/upgrade/node_verification.yml | 2 +- .../tasks/upgrade/post_rollback.yml | 5 - .../tasks/upgrade/post_upgrade.yml | 8 - .../tasks/upgrade/pre_rollback.yml | 5 - .../tasks/upgrade/pre_upgrade.yml | 201 --------- ansible-scylla-node/tasks/upgrade/upgrade.yml | 383 ++++++++++++++++++ .../tasks/upgrade/upgrade_debian.yml | 49 --- .../tasks/upgrade/upgrade_redhat.yml | 49 --- .../tasks/upgrade/upgrade_ubuntu.yml | 49 --- 10 files changed, 536 insertions(+), 469 deletions(-) delete mode 100644 ansible-scylla-node/tasks/upgrade/post_rollback.yml delete mode 100644 ansible-scylla-node/tasks/upgrade/post_upgrade.yml delete mode 100644 ansible-scylla-node/tasks/upgrade/pre_rollback.yml delete mode 100644 ansible-scylla-node/tasks/upgrade/pre_upgrade.yml create mode 100644 ansible-scylla-node/tasks/upgrade/upgrade.yml delete mode 100644 ansible-scylla-node/tasks/upgrade/upgrade_debian.yml delete mode 100644 ansible-scylla-node/tasks/upgrade/upgrade_redhat.yml delete mode 100644 ansible-scylla-node/tasks/upgrade/upgrade_ubuntu.yml diff --git a/ansible-scylla-node/tasks/upgrade/main.yml b/ansible-scylla-node/tasks/upgrade/main.yml index f78bc908..02bba845 100644 --- a/ansible-scylla-node/tasks/upgrade/main.yml +++ b/ansible-scylla-node/tasks/upgrade/main.yml @@ -78,6 +78,11 @@ } } +# Networking +- name: Resolve a scylla_listen_address as a fact + set_fact: + listen_address: "{{ scylla_listen_address }}" + # Scylla information - name: Gather information about Scylla installation @@ -92,7 +97,9 @@ edition: "{{ edition_installed }}", edition_friendly_name: "{{ 'Open Source' if is_oss_installed else 'Enterprise' }}", package_name: "{{ package_installed }}", - version: "{{ ansible_facts.packages[package_installed][0].version }}", + # Debian variants will return version == -n..-n + # RHEL/CentOS will return version == + version: "{{ (ansible_facts.packages[package_installed][0].version).split('-')[0] }}", major_version: "{{ (ansible_facts.packages[package_installed][0].version).split('.')[0] }}.{{ (ansible_facts.packages[package_installed][0].version).split('.')[1] }}" } when: is_scylla_installed @@ -101,12 +108,13 @@ ansible.builtin.fail: msg: "Scylla was not detected." when: scylla_detected is not defined - + any_errors_fatal: True # Scylla installation validation - name: Check existing Scylla installation operating system incompatibility ansible.builtin.fail: msg: "{{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }} is not supported." + any_errors_fatal: True when: > ansible_facts['distribution'] |lower not in scylla_support or scylla_support[ansible_facts['distribution'] |lower][ansible_facts['distribution_major_version']] is not defined or @@ -116,10 +124,10 @@ ansible.builtin.fail: msg: "Scylla {{ scylla_detected['edition_friendly_name'] }} {{ scylla_detected['version'] }} is not supported." when: scylla_detected['major_version'] not in scylla_support[ansible_facts['distribution'] |lower][ansible_facts['distribution_major_version']][scylla_detected['edition']] + any_errors_fatal: True # Scylla upgrade information -# ----- TODO: CHANGE THIS TASK TO ALLOW 'latest' FOR MINOR UPGRADES ----- - name: Gather information about Scylla upgrade vars: is_oss_selected: "{{ true if scylla_edition == 'oss' else false }}" @@ -135,13 +143,13 @@ upgrade: true } - # Scylla version validation - name: Check if Scylla version was specified incorrectly vars: version_format: "{{ 'X.Y.Z' if scylla_upgrade['edition'] else 'UVWX.Y.Z' }}" ansible.builtin.fail: msg: "Version {{ scylla_upgrade['version'] }} specified for Scylla {{ scylla_upgrade['edition_friendly_name'] }} is incomplete and can't be used for a minor upgrade. Version format is: '{{ version_format }}'." + any_errors_fatal: True when: - scylla_version != 'latest' - scylla_upgrade['version'] == scylla_upgrade['major_version'] @@ -150,26 +158,28 @@ - name: Check if specified Scylla version can not be used as major upgrade ansible.builtin.fail: msg: "Version {{ scylla_upgrade['version'] }} specified for Scylla {{ scylla_upgrade['edition_friendly_name'] }} can't be used as a major upgrade since version detected is {{ scylla_detected['version'] }}." + any_errors_fatal: True when: > - upgrade_major and + scylla_version != 'latest' and upgrade_major and (scylla_detected['version'] == scylla_upgrade['version'] or - scylla_detected['major_version'] == scylla_upgrade['major_version']) + scylla_detected['major_version'] == scylla_upgrade['major_version']) - name: Check if specified Scylla version can not be used as minor upgrade ansible.builtin.fail: msg: "Version {{ scylla_upgrade['version'] }} specified for Scylla {{ scylla_upgrade['edition_friendly_name'] }} can't be used as a minor upgrade since version detected is {{ scylla_detected['version'] }}." + any_errors_fatal: True when: > not upgrade_major and (scylla_detected['version'] == scylla_upgrade['version'] or scylla_detected['major_version'] != scylla_upgrade['major_version']) - # Scylla upgrade validation and downgrade protection - name: Check Scylla upgrade validation and downgrade protection block: - name: Check Scylla upgrade edition and version compatibility ansible.builtin.fail: msg: "Scylla {{ scylla_upgrade['edition_friendly_name'] }} {{ scylla_upgrade['major_version'] }} is not supported, thus it can't be used as upgrade." + any_errors_fatal: True when: - scylla_upgrade['major_version'] not in scylla_support[ansible_facts['distribution'] |lower][ansible_facts['distribution_major_version']][scylla_upgrade['edition']] @@ -186,120 +196,160 @@ id: "{{ scylla_upgrade['id'] }}", upgrade: false } - when: upgraded_version is version(detected_version,'<') + when: upgraded_version is version(detected_version,'<') and scylla_version != 'latest' - name: Check if downgrading is not allowed ansible.builtin.fail: - msg: "Downgrade feature it not implemented yet." - # vars: - # detected_version: "{{ scylla_detected['major_version'] if upgrade_major else scylla_detected['version'] }}" - # upgraded_version: "{{ scylla_upgrade['major_version'] if upgrade_major else scylla_upgrade['version'] }}" - # ansible.builtin.fail: - # msg: "Scylla {{ scylla_detected['edition_friendly_name'] }} {{ detected_version }} is installed and you are trying to downgrade to {{ upgraded_version }}, but downgrading is not allowed." + msg: > + "{{ scylla_upgrade['version'] }} is lower or equal than currently installed {{ scylla_detected['version'] }}. + If you know what you are doing, specify `upgrade_allow_user_manual_downgrade: True' variable in your main playbook" + any_errors_fatal: True when: - not scylla_upgrade['upgrade'] - not upgrade_allow_user_manual_downgrade - - name: Set baseline about versions - ansible.builtin.set_fact: - scylla_upgrade_version_fix: false - - - name: Verify relation between versions - vars: - distro_name: "{{ ansible_facts['distribution'] |lower }}" - distro_version: "{{ ansible_facts['distribution_major_version'] }}" - detected_version_index: "{{ lookup('ansible.utils.index_of', scylla_support[distro_name][distro_version][scylla_detected['edition']], 'eq', scylla_detected['major_version']) }}" - upgrade_version_index: "{{ lookup('ansible.utils.index_of', scylla_support[distro_name][distro_version][scylla_upgrade['edition']], 'eq', scylla_upgrade['major_version']) }}" - ansible.builtin.set_fact: - scylla_upgrade_version_fix: true - when: (upgrade_version_index |int - detected_version_index|int) |abs > 1 - - - name: Check if the upgrade can not be straightly applied - ansible.builtin.fail: - msg: "Version {{ scylla_detected['major_version'] }} can't be upgraded straightly to {{ scylla_upgrade['major_version'] }}." - when: - - scylla_upgrade_version_fix - - not upgrade_autocorrect_version - - - name: Check the next available upgrade for existing Scylla installation - vars: - distro_name: "{{ ansible_facts['distribution'] |lower }}" - distro_version: "{{ ansible_facts['distribution_major_version'] }}" - index_delta: "{{ -1 if scylla_upgrade['upgrade'] else 1 }}" - detected_version_index: "{{ lookup('ansible.utils.index_of', scylla_support[distro_name][distro_version][scylla_detected['edition']], 'eq', scylla_detected['major_version']) }}" - next_available_upgrade_version_index: "{{ detected_version_index |int + index_delta |int }}" - next_available_upgrade_version: "{{ scylla_support[distro_name][distro_version][scylla_upgrade['edition']][next_available_upgrade_version_index |int:] |first }}" - ansible.builtin.set_fact: - scylla_upgrade: { - edition: "{{ scylla_upgrade['edition'] }}", - edition_friendly_name: "{{ scylla_upgrade['edition_friendly_name'] }}", - version: "{{ next_available_upgrade_version }}", - major_version: "{{ next_available_upgrade_version.split('.')[0] }}.{{ next_available_upgrade_version.split('.')[1] }}", - id: "{{ scylla_upgrade['id'] }}", - upgrade: "{{ scylla_upgrade['upgrade'] }}" - } - when: - - scylla_upgrade_version_fix - - upgrade_autocorrect_version - - next_available_upgrade_version_index |int >= 0 - - next_available_upgrade_version_index |int < (scylla_support[distro_name][distro_version][scylla_upgrade['edition']] | length) - - - name: Check if the upgrade version was automatically modified - ansible.builtin.debug: - msg: "Scylla upgrade version was modified to {{ scylla_upgrade['version'] }}" - when: scylla_upgrade_version_fix - when: scylla_upgrade['edition'] == scylla_detected['edition'] - - # ----- TODO: Scylla cross-edition upgrade validation and downgrade protection ----- - name: Check Scylla cross-edition upgrade validation and downgrade protection ansible.builtin.fail: msg: "Cross-edition downgrade/upgrade is not implemented yet." + any_errors_fatal: True when: - scylla_upgrade['edition'] != scylla_detected['edition'] - upgrade_major - upgrade_cross_edition +# Determine whether an upgrade actually exists to its target release +- name: Determine available Enterprise updates for RedHat variants + shell: + cmd: yum list available scylla-enterprise | egrep ^scylla-enterprise | awk '{ print $2 }' + register: latest_release + when: "'scylla-enterprise' in ansible_facts.packages and ansible_os_family == 'RedHat'" + +- set_fact: latest_detected_release={{ latest_release.stdout }} + when: latest_release.changed + +- name: Determine available OSS updates for RedHat variants + shell: + cmd: yum list available scylla | egrep ^scylla | awk '{ print $2 }' + register: latest_release + when: "'scylla' in ansible_facts.packages and ansible_os_family == 'RedHat'" + +- set_fact: latest_detected_release={{ latest_release.stdout }} + when: latest_release.changed + +# This check is necessary because when we are on latest under RHEL variants, `yum list available' will return an empty string +# therefore we pin the fact to its detected_version early on +- set_fact: latest_detected_release={{ scylla_detected['version'] }} + when: ansible_os_family == 'RedHat' and latest_detected_release | length == 0 + +- name: Determine available Enterprise updates for Debian variants + shell: + cmd: apt-cache madison scylla-enterprise | egrep ^scylla-enterprise | awk -F'|' '{ print $2 }' | head -1 | awk '{ print $1 }' + register: latest_release + when: "'scylla-enterprise' in ansible_facts.packages and ansible_os_family == 'Debian'" + +- set_fact: latest_detected_release={{ latest_release.stdout }} + when: latest_release.changed + +- name: Determine available OSS updates for Debian variants + shell: + cmd: apt-cache madison scylla | egrep ^scylla | awk -F'|' '{ print $2 }' | head -1 | awk '{ print $1 }' + register: latest_release + when: "'scylla' in ansible_facts.packages and ansible_os_family == 'Debian'" + +- set_fact: latest_detected_release={{ latest_release.stdout }} + when: latest_release.changed + +- name: Fail when requested version is not within the last release available + fail: + msg: "Requested upgrade to {{ scylla_upgrade['version'] }} but latest found in repository is {{ latest_detected_release }}" + any_errors_fatal: True + when: not upgrade_major and scylla_upgrade['version'] is version(latest_detected_release,'>=') and scylla_version != 'latest' + +- name: Fail when latest is specified and we are already on latest + fail: + msg: "'latest' was specified, but currently installed version {{ scylla_detected['version'] }} is the same as last available {{ latest_detected_release }}" + any_errors_fatal: True + when: not upgrade_major and scylla_detected['version'] is version(latest_detected_release,'>=') and scylla_version == 'latest' + +# If we are downgrading, build list of packages to downgrade +# RHEL Specific +- name: Create list of packages to downgrade + set_fact: + installed_pkgs: "{{ packages | dict2items | selectattr('key', 'match', '^' + scylla_detected['package_name']) | list | items2dict }}" + pkg_list: [] + when: not scylla_upgrade['upgrade'] + +- name: Append results to list of packages + set_fact: + pkg_list: "{{ pkg_list + [ item.key + '-' + scylla_upgrade['version'] ] }}" + loop: "{{ installed_pkgs | dict2items }}" + when: not scylla_upgrade['upgrade'] and ansible_os_family == 'RedHat' + +# Debian specific +- name: Set Scylla package prefix as OSS + set_fact: + scylla_package_prefix: "scylla" + when: scylla_edition == 'oss' and ansible_os_family == 'Debian' + +- name: Set Scylla package prefix as Enterprise + set_fact: + scylla_package_prefix: "scylla-enterprise" + when: scylla_edition == 'enterprise' and ansible_os_family == 'Debian' + +- name: "Upgrade Scylla to {{ scylla_upgrade['version'] }}" + set_fact: + scylla_version_to_install: "{{ scylla_upgrade['version'] }}" + when: + - scylla_version != 'latest' + - ansible_os_family == 'Debian' -# Upgrade -- name: Trigger pre-upgrade tasks - include_tasks: pre_upgrade.yml +- name: "Upgrade Scylla to latest" + set_fact: + scylla_version_to_install: "latest" + when: + - scylla_version == 'latest' + - ansible_os_family == 'Debian' -- name: Trigger operating system specific upgrade tasks - include_tasks: "upgrade_{{ scylla_support[ansible_facts['distribution'] |lower][ansible_facts['distribution_major_version']]['template'] }}.yml" +- name: Get versions of {{ scylla_edition }} package + shell: apt list -a {{ scylla_package_prefix }} 2>/dev/null | grep `dpkg --print-architecture` | awk '{split($0,a," "); print a[2]}' | egrep -v "^{{ scylla_version_escaped }}[0123456789]+" | egrep "^{{ scylla_version_escaped }}" | sort | uniq + register: aptversions + vars: + scylla_version_escaped: "{{ scylla_version_to_install | regex_escape }}" + when: ansible_os_family == 'Debian' -- name: Trigger post-upgrade tasks - include_tasks: post_upgrade.yml +- name: "Validate scylla version correctness" + ansible.builtin.fail: + msg: "Too many/few choices for a requested version '{{ scylla_version_to_install }}': {{ aptversions.stdout_lines }}. Bailing out!" + when: ansible_os_family == 'Debian' and aptversions.stdout_lines | length != 1 and scylla_version_to_install != 'latest' + any_errors_fatal: True +- name: Fetch version parts of Scylla package + set_fact: + scylla_version_split: "{{ aptversions.stdout | regex_findall(regexp, ignorecase=True) }}" + vars: + # All we know that the version is a string comprised of 3 parts separated by '-' + regexp: '^([^\-]+)-([^\-]+)-([^\-]+)$' + when: ansible_os_family == 'Debian' + +- name: Append results to list of packages + set_fact: + pkg_list: "{{ pkg_list + [ item.key + '=' + aptversions.stdout ] }}" + loop: "{{ installed_pkgs | dict2items }}" + when: not scylla_upgrade['upgrade'] and ansible_os_family == 'Debian' + +# Upgrade +- name: Trigger pre-upgrade tasks + run_once: True + include_tasks: upgrade.yml + loop: "{{ groups['scylla'] }}" + loop_control: + loop_var: delegate_host # Upgrade verification -- name: Upgrade verification - block: - - name: "Wait {{ upgrade_break_before_verification }} seconds before verifying the upgrade" - ansible.builtin.wait_for: - timeout: "{{ upgrade_break_before_verification |int }}" - - - name: Verify if the upgrade was done sucessfully - include_tasks: node_verification.yml - rescue: - # Rollback - - name: Trigger pre-rollback tasks - include_tasks: pre_rollback.yml - - #- name: Trigger operating system specific rollback tasks - # include_tasks: "rollback_{{ scylla_support[ansible_facts['distribution'] |lower][ansible_facts['distribution_major_version']]['template'] }}.yml" - - #- name: Trigger post-rollback tasks - # include_tasks: post_rollback.yml - - # Rollback verification - - name: Wait {{ upgrade_break_before_verification }} seconds before verifying the rollback - ansible.builtin.wait_for: - timeout: "{{ upgrade_break_before_verification |int }}" - - - name: Verify if the upgrade was done sucessfully - include_tasks: node_verification.yml - - - # ----- TODO: If the upgrade version is the latest one and if 'latest' was used to specify the version - # replace upgrade version with latest +- name: "Wait {{ upgrade_break_before_verification }} seconds before verifying the upgrade" + ansible.builtin.wait_for: + timeout: "{{ upgrade_break_before_verification |int }}" + +- name: Verify if the upgrade was done sucessfully + include_tasks: node_verification.yml diff --git a/ansible-scylla-node/tasks/upgrade/node_verification.yml b/ansible-scylla-node/tasks/upgrade/node_verification.yml index 5e031d50..1a569df0 100644 --- a/ansible-scylla-node/tasks/upgrade/node_verification.yml +++ b/ansible-scylla-node/tasks/upgrade/node_verification.yml @@ -62,4 +62,4 @@ version_installed: "{{ version_major_installed if upgrade_major else version_full_installed }}" ansible.builtin.fail: msg: "Version set ({{ version_specified }}) and version installed ({{ version_installed }}) are different, thus the upgrade failed." - when: version_specified != version_installed + when: version_specified != version_installed and scylla_version != 'latest' diff --git a/ansible-scylla-node/tasks/upgrade/post_rollback.yml b/ansible-scylla-node/tasks/upgrade/post_rollback.yml deleted file mode 100644 index 4351acb7..00000000 --- a/ansible-scylla-node/tasks/upgrade/post_rollback.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Feature not implemented - ansible.builtin.fail: - msg: "This feature is not implemented yet" diff --git a/ansible-scylla-node/tasks/upgrade/post_upgrade.yml b/ansible-scylla-node/tasks/upgrade/post_upgrade.yml deleted file mode 100644 index c1abf9c5..00000000 --- a/ansible-scylla-node/tasks/upgrade/post_upgrade.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - -# Start Scylla service -- name: Start Scylla service - ansible.builtin.service: - name: scylla-server - state: started - become: true diff --git a/ansible-scylla-node/tasks/upgrade/pre_rollback.yml b/ansible-scylla-node/tasks/upgrade/pre_rollback.yml deleted file mode 100644 index 4351acb7..00000000 --- a/ansible-scylla-node/tasks/upgrade/pre_rollback.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Feature not implemented - ansible.builtin.fail: - msg: "This feature is not implemented yet" diff --git a/ansible-scylla-node/tasks/upgrade/pre_upgrade.yml b/ansible-scylla-node/tasks/upgrade/pre_upgrade.yml deleted file mode 100644 index 4f1d5d3e..00000000 --- a/ansible-scylla-node/tasks/upgrade/pre_upgrade.yml +++ /dev/null @@ -1,201 +0,0 @@ ---- - -# Check cluster schema -- name: Get cluster schema - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_proxy/schema_versions" - method: GET - register: cluster_schemas - until: cluster_schemas.status == 200 and cluster_schemas.json |length == 1 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Check if all cluster nodes use different schemas - ansible.builtin.fail: - msg: "Cluster schema is out of synchronization. Please check each node of the cluster." - when: cluster_schemas.json |length > 1 - - -# Disable native transport -- name: Get current native transport status - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/native_transport" - method: GET - register: node_binary_status - until: node_binary_status.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Disable native transport - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/native_transport" - method: DELETE - register: node_binary_disable - until: node_binary_disable.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Get current native transport status after disable it - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/native_transport" - method: GET - register: node_binary_status - until: node_binary_status.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Check if native transport is still enabled - ansible.builtin.fail: - msg: "Native transport it not disabled due to an unknown reason. Please check the node." - when: node_binary_status.json |bool - - -# Disable gossip -- name: Get current gossip status - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/gossiping" - method: GET - register: node_gossip_status - until: node_gossip_status.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Disable gossip - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/gossiping" - method: DELETE - register: node_gossip_disable - until: node_gossip_disable.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Get current gossip status after disable it - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/gossiping" - method: GET - register: node_gossip_status - until: node_gossip_status.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Check if gossip is still enabled - ansible.builtin.fail: - msg: "Gossip it not disabled due to an unknown reason. Please check the node." - when: node_gossip_status.json |bool - - -# Drain the node -- name: Drain the node - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/drain" - method: POST - register: node_drain_enable - until: node_drain_enable.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - -- name: Ensure the node is fully drained - block: - - name: Ensure the node is fully drained - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/drain" - method: GET - register: node_drain_status - until: node_drain_status.json.find("Drained") != -1 - retries: "{{ upgrade_api_default_retries |int * 3 }}" - delay: "{{ upgrade_api_default_delay |int }}" - rescue: - # Kill Scylla process - - name: Kill Scylla process - ansible.builtin.shell: killall --exact --signal SIGKILL scylla - register: scylla_process_kill - become: true - - # Enable snapshot generation - - name: Enable snapshot generation - ansible.builtin.debug: - msg: "Snapshot generation was set to be disabled but node draining failed. Enabling snapshot generation." - when: not upgrade_generate_snapshots - - -# Prepare and create snapshot -- name: Prepare and create snapshot - block: - - name: Create snapshot - ansible.builtin.uri: - url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/snapshots?tag={{ scylla_upgrade['id'] }}" - method: POST - register: node_snapshot_create - until: node_snapshot_create.status == 200 - retries: "{{ upgrade_api_default_retries |int }}" - delay: "{{ upgrade_api_default_delay |int }}" - - - name: Print snapshot identifier used - ansible.builtin.debug: - msg: "Snapshot identifier: {{ scylla_upgrade['id'] }}" - when: upgrade_generate_snapshots or scylla_process_kill is defined - - -# Stop Scylla service -- name: Stop Scylla service - ansible.builtin.service: - name: scylla-server - state: stopped - become: true - - -# Determine the folder name for 'sysconfig/default' files -- name: Determine the folder name for distribution-specific files - vars: - is_etc_defaults: "{{ true if ansible_facts['distribution'] |lower == 'debian' or ansible_facts['distribution'] |lower == 'ubuntu' else false }}" - ansible.builtin.set_fact: - system_config_folder: "{{ 'default' if is_etc_defaults else 'sysconfig' }}" - - -# Prepare and create backup -- name: Create backup folders structure - ansible.builtin.file: - path : "{{ item }}" - state: directory - mode: '0755' - become: true - loop: - - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}" - - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc" - - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/scylla.d" - - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/{{ system_config_folder }}" - -- name: Dump Scylla version - ansible.builtin.copy: - content: "{{ scylla_detected['version'] }}" - dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/scylla_version.txt" - become: true - -- name: Backup Scylla configuration file - ansible.builtin.copy: - src: /etc/scylla/scylla.yaml - dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/scylla.yaml" - mode: preserve - remote_src: yes - become: true - -- name: Backup Scylla extra configuration files - ansible.builtin.copy: - src: /etc/scylla.d/ - dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/scylla.d/" - directory_mode: yes - mode: preserve - remote_src: yes - become: true - -- name: Backup Scylla system-wide configuration files - ansible.builtin.copy: - src: "/etc/{{ system_config_folder }}/{{ item }}" - dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/{{ system_config_folder }}/{{ item }}" - directory_mode: yes - mode: preserve - remote_src: yes - become: true - loop: - - "scylla-server" - - "scylla-jmx" diff --git a/ansible-scylla-node/tasks/upgrade/upgrade.yml b/ansible-scylla-node/tasks/upgrade/upgrade.yml new file mode 100644 index 00000000..56f40604 --- /dev/null +++ b/ansible-scylla-node/tasks/upgrade/upgrade.yml @@ -0,0 +1,383 @@ +--- + +# Check cluster schema +- name: Get cluster schema + ansible.builtin.uri: + url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_proxy/schema_versions" + method: GET + register: cluster_schemas + until: cluster_schemas.status == 200 and cluster_schemas.json |length == 1 + retries: "{{ upgrade_api_default_retries |int }}" + delay: "{{ upgrade_api_default_delay |int }}" + delegate_to: "{{ delegate_host }}" + +- name: Check if schema is in agreement + ansible.builtin.fail: + msg: "Schema disagreement detected. " + when: cluster_schemas.json |length > 1 + delegate_to: "{{ delegate_host }}" + +# Drain the node +- name: Drain the node + ansible.builtin.uri: + url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/drain" + method: POST + register: node_drain_enable + until: node_drain_enable.status == 200 + retries: "{{ upgrade_api_default_retries |int }}" + delay: "{{ upgrade_api_default_delay |int }}" + delegate_to: "{{ delegate_host }}" + +- name: Ensure the node is fully drained + ansible.builtin.uri: + url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/drain" + method: GET + register: node_drain_status + until: node_drain_status.json.find("Drained") != -1 + retries: "{{ upgrade_api_default_retries |int * 3 }}" + delay: "{{ upgrade_api_default_delay |int }}" + delegate_to: "{{ delegate_host }}" + +# Prepare and create snapshot +- name: Create snapshot + ansible.builtin.uri: + url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/storage_service/snapshots?tag={{ scylla_upgrade['id'] }}" + method: POST + register: node_snapshot_create + until: node_snapshot_create.status == 200 + retries: "{{ upgrade_api_default_retries |int }}" + delay: "{{ upgrade_api_default_delay |int }}" + delegate_to: "{{ delegate_host }}" + +- name: Print snapshot identifier used + ansible.builtin.debug: + msg: "Snapshot identifier: {{ scylla_upgrade['id'] }}" + when: upgrade_generate_snapshots or scylla_process_kill is defined + delegate_to: "{{ delegate_host }}" + +# Stop Scylla service +- name: Stop Scylla service + ansible.builtin.service: + name: scylla-server + state: stopped + become: true + delegate_to: "{{ delegate_host }}" + +# Determine the folder name for 'sysconfig/default' files +- name: Determine the folder name for distribution-specific files + vars: + is_etc_defaults: "{{ true if hostvars[delegate_host]['ansible_os_family'] |lower == 'debian' else false }}" + ansible.builtin.set_fact: + system_config_folder: "{{ 'default' if is_etc_defaults else 'sysconfig' }}" + +# Prepare and create backup +- name: Create backup folders structure + ansible.builtin.file: + path : "{{ item }}" + state: directory + mode: '0755' + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + loop: + - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}" + - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc" + - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/scylla.d" + - "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/{{ system_config_folder }}" + +- name: Dump Scylla version + ansible.builtin.copy: + content: "{{ scylla_detected['version'] }}" + dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/scylla_version.txt" + become: true + delegate_to: "{{ delegate_host }}" + +- name: Backup Scylla configuration file + ansible.builtin.copy: + src: /etc/scylla/scylla.yaml + dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/scylla.yaml" + mode: preserve + remote_src: yes + become: true + delegate_to: "{{ delegate_host }}" + +- name: Backup Scylla extra configuration files + ansible.builtin.copy: + src: /etc/scylla.d/ + dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/scylla.d/" + directory_mode: yes + mode: preserve + remote_src: yes + become: true + delegate_to: "{{ delegate_host }}" + +- name: Backup Scylla system-wide configuration files + ansible.builtin.copy: + src: "/etc/{{ system_config_folder }}/{{ item }}" + dest: "{{ upgrade_backup_path }}/{{ scylla_upgrade['id'] }}/etc/{{ system_config_folder }}/{{ item }}" + directory_mode: yes + mode: preserve + remote_src: yes + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + loop: + - "scylla-server" + - "scylla-jmx" + +# RHEL variants +- name: Find existing Scylla repositories + find: + paths: /etc/yum.repos.d + patterns: 'scylla*.repo' + file_type: 'file' + register: existing_repos + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'RedHat' + +- name: Remove existing Scylla repositories + file: + path: "{{ item['path'] }}" + state: absent + with_items: "{{ hostvars[delegate_host]['existing_repos']['files'] }}" + notify: Clean yum metadata + become: true + when: upgrade_major and hostvars[delegate_host]['ansible_os_family'] == 'RedHat' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Recreate Scylla repositories file +- name: Recreate Scylla repositories file + get_url: + url: "{{ item }}" + dest: '/etc/yum.repos.d/{{ item.split("/")[-1] | lower }}' + mode: '0644' + owner: root + with_items: "{{ scylla_rpm_repos }}" + when: item.split(".")[-1] == "repo" and upgrade_major and hostvars[delegate_host]['ansible_os_family'] == 'RedHat' + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Ensure 'ABRT' is not installed +- name: Ensure 'ABRT' is not installed + ansible.builtin.yum: + name: 'abrt' + state: absent + lock_timeout: 60 + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'RedHat' + +# Check if Scylla packages are excluded and remove them from the list +- name: Check if Scylla packages are excluded and remove them from the list + ansible.builtin.replace: + path: /etc/yum.conf + regexp: 'scylla([*]|[\s.*]|[-]\w+)' + replace: '' + become: true + when: upgrade_major and hostvars[delegate_host]['ansible_os_family'] == 'RedHat' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Upgrade Scylla +- name: Upgrade Scylla to {{ scylla_upgrade['version'] }} + yum: + name: "{{ scylla_detected['package_name'] }}-{{ scylla_upgrade['version'] }}*" + state: present + update_cache: yes + lock_timeout: 60 + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'RedHat' and scylla_version != 'latest' and hostvars[delegate_host]['scylla_upgrade']['upgrade'] + +- name: Upgrade Scylla to latest + yum: + name: "{{ scylla_detected['package_name'] }}" + state: latest + update_cache: yes + lock_timeout: 60 + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'RedHat' and scylla_version == 'latest' and hostvars[delegate_host]['scylla_upgrade']['upgrade'] + +- name: UNSAFE - Downgrade Scylla + yum: + name: "{{ hostvars[delegate_host]['pkg_list'] }}" + state: present + allow_downgrade: True + update_cache: yes + lock_timeout: 60 + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'RedHat' and scylla_version != 'latest' and not hostvars[delegate_host]['scylla_upgrade']['upgrade'] + + +# Debian variants +# Ensure apt-transport-https is installed +- name: Ensure apt-transport-https is installed + apt: + name: apt-transport-https + state: present + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' + +# Find and delete existing repos +- name: Find Scylla repositories + find: + paths: /etc/apt/sources.list.d + patterns: 'scylla*.list' + file_type: 'file' + register: existing_repos + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' + +- name: Remove existing Scylla repositories + file: + path: "{{ item['path'] }}" + state: absent + with_items: "{{ hostvars[delegate_host]['existing_repos']['files'] }}" + become: true + when: upgrade_major and hostvars[delegate_host]['ansible_os_family'] == 'Debian' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Add Scylla apt key +- name: Add an apt key by id from a keyserver + apt_key: + keyserver: "{{ scylla_repo_keyserver }}" + id: "{{ item }}" + state: present + with_items: "{{ scylla_repo_keys }}" + when: upgrade_major and install_type == 'online' and scylla_repo_keyserver is defined and scylla_repo_keys is defined and (scylla_repo_keys|length > 0) and hostvars[delegate_host]['ansible_os_family'] == 'Debian' + become: True + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +- name: Add an Apt signing key for Debian + apt_key: + url: "{{ item }}" + state: present + with_items: "{{ scylla_repo_keyfile_urls }}" + when: upgrade_major and scylla_repo_keyfile_urls is defined and (scylla_repo_keyfile_urls|length > 0) and hostvars[delegate_host]['ansible_os_family'] == 'Debian' + become: True + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +- name: Install Scylla repo + get_url: + url: "{{ item }}" + dest: '/etc/apt/sources.list.d/{{ item.split("/")[-1] | lower }}' + mode: '0644' + owner: root + with_items: "{{ scylla_deb_repos }}" + when: upgrade_major and install_type == 'online' and item.split(".")[-1] == "list" and hostvars[delegate_host]['ansible_os_family'] == 'Debian' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Unhold Scylla package in case it's marked as 'hold' +- name: Unhold Scylla package + ansible.builtin.dpkg_selections: + name: "{{ scylla_detected['package_name'] }}*" + selection: install + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' + +# Upgrade Scylla +- name: Nuke an OSS pin file if exists + file: + state: absent + path: /etc/apt/preferences.d/99-scylla + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +- name: Nuke an Enterprise pin file if exists + file: + state: absent + path: /etc/apt/preferences.d/99-scylla-enterprise + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +- name: Install latest OSS Scylla + apt: + name: scylla + state: latest + update_cache: True + when: scylla_edition == 'oss' and scylla_version_to_install == 'latest' and hostvars[delegate_host]['ansible_os_family'] == 'Debian' and hostvars[delegate_host]['scylla_upgrade']['upgrade'] + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +- name: Install latest Enterprise Scylla + apt: + name: scylla-enterprise + state: latest + update_cache: True + when: scylla_edition == 'enterprise' and scylla_version_to_install == 'latest' and hostvars[delegate_host]['ansible_os_family'] == 'Debian' and hostvars[delegate_host]['scylla_upgrade']['upgrade'] + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Debian +# In a block to prevent https://github.com/ansible/ansible/issues/58835 +# As upgrading Ansible can (and often do) break other aspects, leave it as is for now. + +- name: Check if package version pin file is required + block: + - name: Create package version pin file + template: + src: templates/apt-pin-file.j2 + dest: "/etc/apt/preferences.d/99-{{ scylla_package_prefix }}" + owner: root + group: root + mode: '0644' + delegate_to: "{{ delegate_host }}" + delegate_facts: false + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' and scylla_version_to_install != 'latest' + when: hostvars[delegate_host]['ansible_os_family'] == 'Debian' and scylla_version_to_install != 'latest' + +- name: "Install {{ aptversions.stdout }}" + apt: + name: "{{ scylla_package_prefix }}={{ aptversions.stdout }}" + state: present + allow_downgrade: no + when: scylla_version_to_install != 'latest' and hostvars[delegate_host]['ansible_os_family'] == 'Debian' and hostvars[delegate_host]['scylla_upgrade']['upgrade'] + become: true + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +- name: UNSAFE - Downgrade Scylla + apt: + name: "{{ hostvars[delegate_host]['pkg_list'] }}" + state: present + allow_downgrade: yes + when: scylla_version_to_install != 'latest' and hostvars[delegate_host]['ansible_os_family'] == 'Debian' and not hostvars[delegate_host]['scylla_upgrade']['upgrade'] + become: True + delegate_to: "{{ delegate_host }}" + delegate_facts: false + +# Post upgrade +# Start Scylla service +- name: Start Scylla service + ansible.builtin.service: + name: scylla-server + state: started + become: true + delegate_to: "{{ delegate_host }}" + +# Wait for at most 2 hours for a node to re-start - reshaping (for instance) can take quite long +- name: "Wait for CQL port" + wait_for: + port: 9042 + host: "{{ hostvars[delegate_host]['listen_address'] }}" + timeout: 7200 + delegate_to: "{{ delegate_host }}" + diff --git a/ansible-scylla-node/tasks/upgrade/upgrade_debian.yml b/ansible-scylla-node/tasks/upgrade/upgrade_debian.yml deleted file mode 100644 index 7c439696..00000000 --- a/ansible-scylla-node/tasks/upgrade/upgrade_debian.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -# Ensure apt-transport-https and dirmngr are installed -- name: Ensure apt-transport-https and dirmngr are installed - ansible.builtin.apt: - name: [apt-transport-https,dirmngr] - state: present - become: true - - -# Add Scylla apt key -- name: Add Scylla apt key - ansible.builtin.apt_key: - keyserver: hkp://keyserver.ubuntu.com:80 - id: 5e08fbd8b5d6ec9c - become: true - - -# Remove existing Scylla repositories -- name: Remove existing Scylla repositories - ansible.builtin.file: - path: /etc/apt/sources.list.d/scylla.list - state: absent - become: true - - -# Create Scylla repositories file -- name: Create Scylla repositories file - ansible.builtin.get_url: - url: "http://downloads.scylladb.com/deb/debian/scylla-{{ scylla_upgrade['major_version'] }}-{{ ansible_facts['distribution_release'] }}.list" - dest: /etc/apt/sources.list.d/scylla.list - mode: '0644' - become: true - - -# Unhold Scylla package in case it's marked as 'hold' -- name: Unhold Scylla package - ansible.builtin.dpkg_selections: - name: "{{ scylla_detected['package_name'] }}*" - selection: install - - -# Upgrade Scylla -- name: "Upgrade Scylla to {{ scylla_upgrade['version'] }}" - set_fact: - scylla_version_to_install: "{{ scylla_upgrade['version'] }}" - -- name: Install Scylla packages - include_tasks: ../Debian_install.yml \ No newline at end of file diff --git a/ansible-scylla-node/tasks/upgrade/upgrade_redhat.yml b/ansible-scylla-node/tasks/upgrade/upgrade_redhat.yml deleted file mode 100644 index de765bdc..00000000 --- a/ansible-scylla-node/tasks/upgrade/upgrade_redhat.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -# Remove existing Scylla repositories -- name: Remove existing Scylla repositories - ansible.builtin.file: - path: /etc/yum.repos.d/scylla.repo - state: absent - notify: Clean yum metadata - become: true - when: upgrade_major - - -# Create Scylla repositories file -- name: Create Scylla repositories file - ansible.builtin.get_url: - url: "http://downloads.scylladb.com/rpm/centos/scylla-{{ scylla_upgrade['major_version'] }}.repo" - dest: /etc/yum.repos.d/scylla.repo - mode: '0644' - become: true - when: upgrade_major - - -# Ensure 'ABRT' is not installed -- name: Ensure 'ABRT' is not installed - ansible.builtin.yum: - name: 'abrt' - state: absent - lock_timeout: 60 - become: true - - -# Check if Scylla packages are excluded and remove them from the list -- name: Check if Scylla packages are excluded and remove them from the list - ansible.builtin.replace: - path: /etc/yum.conf - regexp: 'scylla([*]|[\s.*]|[-]\w+)' - replace: '' - become: true - when: upgrade_major - - -# Upgrade Scylla -- name: Upgrade Scylla to {{ scylla_upgrade['version'] }} - ansible.builtin.yum: - name: "{{ scylla_detected['package_name'] }}*-{{ scylla_upgrade['version'] }}*" - state: present - update_cache: yes - lock_timeout: 60 - become: true diff --git a/ansible-scylla-node/tasks/upgrade/upgrade_ubuntu.yml b/ansible-scylla-node/tasks/upgrade/upgrade_ubuntu.yml deleted file mode 100644 index 5a38c404..00000000 --- a/ansible-scylla-node/tasks/upgrade/upgrade_ubuntu.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -# Ensure apt-transport-https is installed -- name: Ensure apt-transport-https is installed - ansible.builtin.apt: - name: apt-transport-https - state: present - become: true - - -# Add Scylla apt key -- name: Add Scylla apt key - ansible.builtin.apt_key: - keyserver: hkp://keyserver.ubuntu.com:80 - id: 5e08fbd8b5d6ec9c - become: true - - -# Remove existing Scylla repositories -- name: Remove existing Scylla repositories - ansible.builtin.file: - path: /etc/apt/sources.list.d/scylla.list - state: absent - become: true - - -# Create Scylla repositories file -- name: Create Scylla repositories file - ansible.builtin.get_url: - url: "http://downloads.scylladb.com/deb/ubuntu/scylla-{{ scylla_upgrade['major_version'] }}-{{ ansible_facts['distribution_release'] }}.list" - dest: /etc/apt/sources.list.d/scylla.list - mode: '0644' - become: true - - -# Unhold Scylla package in case it's marked as 'hold' -- name: Unhold Scylla package - ansible.builtin.dpkg_selections: - name: "{{ scylla_detected['package_name'] }}*" - selection: install - - -# Upgrade Scylla -- name: "Upgrade Scylla to {{ scylla_upgrade['version'] }}" - set_fact: - scylla_version_to_install: "{{ scylla_upgrade['version'] }}" - -- name: Install Scylla packages - include_tasks: ../Debian_install.yml \ No newline at end of file