From 2ed780a09ca1745bb9bc30206613fde31046c67c Mon Sep 17 00:00:00 2001 From: Christopher Papke Date: Tue, 27 Feb 2024 14:46:59 -0800 Subject: [PATCH 1/4] working on adding healthcheck to preflight --- modules/aws/sonar-upgrader/main.tf | 2 + .../aws/sonar-upgrader/provision_script.tpl | 6 ++ .../python_upgrader/upgrade/main.py | 66 ++++++++++++++----- modules/aws/sonar-upgrader/variables.tf | 10 +++ 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/modules/aws/sonar-upgrader/main.tf b/modules/aws/sonar-upgrader/main.tf index 53091e614..66ac88f13 100644 --- a/modules/aws/sonar-upgrader/main.tf +++ b/modules/aws/sonar-upgrader/main.tf @@ -12,6 +12,8 @@ locals { # clean_old_deployments = var.clean_old_deployments stop_on_failure = var.stop_on_failure tarball_location = jsonencode(var.tarball_location) + ignore_healthcheck_warning = var.ignore_healthcheck_warnings + ignore_healthcheck_checks = var.ignore_healthcheck_checks }) } diff --git a/modules/aws/sonar-upgrader/provision_script.tpl b/modules/aws/sonar-upgrader/provision_script.tpl index 3f32b9db1..a4356c667 100755 --- a/modules/aws/sonar-upgrader/provision_script.tpl +++ b/modules/aws/sonar-upgrader/provision_script.tpl @@ -15,6 +15,12 @@ PYTHONPATH=${path}/python_upgrader python3 -u -m upgrade.main \ --connection_timeout "${connection_timeout}" \ --test_connection "${test_connection}" \ --run_preflight_validations "${run_preflight_validations}" \ + %{ if ignore_healthcheck_warning ~} + --ignore-healthcheck-warnings \ + %{ endif ~} + %{ for check in ignore_healthcheck_checks ~} + --ignore-healthcheck-check "${check}" \ + %{ endfor ~} --run_upgrade "${run_upgrade}" \ --run_postflight_validations "${run_postflight_validations}" \ --stop_on_failure "${stop_on_failure}" \ diff --git a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py index 97ac18691..9df595f07 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py @@ -310,9 +310,12 @@ def collect_python_location(extended_node): def run_preflight_validations_stage(args, agentless_gw_extended_node_dict, dsf_hub_extended_node_dict, upgrade_status_service): if args.run_preflight_validations: - preflight_validations_passed = run_preflight_validations(args.stop_on_failure, args.target_version, - agentless_gw_extended_node_dict, - dsf_hub_extended_node_dict, upgrade_status_service) + preflight_validations_passed = run_preflight_validations( + args.stop_on_failure, args.target_version, + args.ignore_healthcheck_warnings, + args.ignore_healthcheck_check, + agentless_gw_extended_node_dict, + dsf_hub_extended_node_dict, upgrade_status_service) if preflight_validations_passed: print(f"### Preflight validations passed for all DSF nodes") @@ -383,37 +386,53 @@ def test_connection_to_extended_node(extended_node, stop_on_failure, upgrade_sta return True -def run_preflight_validations(stop_on_failure, target_version, agentless_gw_extended_node_dict, - dsf_hub_extended_node_dict, upgrade_status_service): +def run_preflight_validations( + stop_on_failure, target_version, ignore_healthcheck_warning, + ignore_healthcheck_checks, agentless_gw_extended_node_dict, + dsf_hub_extended_node_dict, upgrade_status_service): print("----- Preflight validations") successful = True for extended_node in chain(agentless_gw_extended_node_dict.values(), dsf_hub_extended_node_dict.values()): if upgrade_status_service.should_run_preflight_validations(extended_node.get('dsf_node_id')): - node_successful = run_preflight_validations_for_node(stop_on_failure, target_version, extended_node, upgrade_status_service) + node_successful = run_preflight_validations_for_node( + stop_on_failure, target_version, + ignore_healthcheck_warning, ignore_healthcheck_checks, + extended_node, upgrade_status_service) successful = successful and node_successful return successful -def run_preflight_validations_for_node(stop_on_failure, target_version, extended_node, upgrade_status_service): +def run_preflight_validations_for_node( + stop_on_failure, target_version, + ignore_healthcheck_warning, ignore_healthcheck_checks, + extended_node, upgrade_status_service): error_message = None try: upgrade_status_service.update_upgrade_status( extended_node.get('dsf_node_id'), UpgradeStatus.RUNNING_PREFLIGHT_VALIDATIONS, ) - preflight_validations_result = run_preflight_validations_script( - target_version, extended_node.get('dsf_node'), extended_node.get('dsf_node_name'), extended_node.get('python_location'), - ) - - if are_preflight_validations_passed(preflight_validations_result): - print(f"### Preflight validations passed for {extended_node.get('dsf_node_name')}") - upgrade_status_service.update_upgrade_status( - extended_node.get('dsf_node_id'), UpgradeStatus.PREFLIGHT_VALIDATIONS_SUCCEEDED, - ) + version_list = extended_node['sysconfig']['JSONAR_VERSION'].split('-')[0].split('.') + version_tuple = tuple(map(int, version_list)) + if version_tuple >= (4, 15): + script_output = run_remote_script(extended_node['dsf_node'], "", f"sudo {extended_node['sysconfig']['JSONAR_BASEDIR']}/bin/health-checker --quiet --output JSON --target-version {target_version}") + results = list(chain.from_iterable(machine['results'] for machine in json.loads(script_output)['machines'])) + results = [result for result in results if result['status'] in ['WARNING', 'FAILURE']] + if ignore_healthcheck_warning: + results = [result for result in results if result['status'] != 'WARNING'] + results = [result for result in results if result['name'] not in ignore_healthcheck_checks] + if results: + error_message = f'Healthchecks failed: {results}' else: - print(f"### Preflight validations didn't pass for {extended_node.get('dsf_node_name')}") - error_message = preflight_validations_result + preflight_validations_result = run_preflight_validations_script( + target_version, extended_node.get('dsf_node'), extended_node.get('dsf_node_name'), extended_node.get('python_location'), + ) + + if not are_preflight_validations_passed(preflight_validations_result): + print(f"### Preflight validations didn't pass for {extended_node.get('dsf_node_name')}") + error_message = preflight_validations_result + except Exception as ex: print(f"### Preflight validations for {extended_node.get('dsf_node_name')} failed with exception: {str(ex)}") error_message = str(ex) @@ -427,6 +446,11 @@ def run_preflight_validations_for_node(stop_on_failure, target_version, extended raise UpgradeException(f"Preflight validations didn't pass for {extended_node.get('dsf_node_id')}") else: return False + else: + print(f"### Preflight validations passed for {extended_node.get('dsf_node_name')}") + upgrade_status_service.update_upgrade_status( + extended_node.get('dsf_node_id'), UpgradeStatus.PREFLIGHT_VALIDATIONS_SUCCEEDED, + ) return True @@ -836,6 +860,12 @@ def get_argument_parser(): parser.add_argument("--run_preflight_validations", type=str_to_bool, default=True, help="Whether to run preflight validations") + parser.add_argument("--ignore-healthcheck-warnings", + default=False, action='store_true', + help="If we should ignore warnings from the healthchecker") + parser.add_argument("--ignore-healthcheck-check", + default=[], action="append", + help="A check that should be ignored, can be added multiple times to ignore multiple checks") parser.add_argument("--run_upgrade", type=str_to_bool, default=True, help="Whether to run the upgrade") parser.add_argument("--run_postflight_validations", type=str_to_bool, default=True, diff --git a/modules/aws/sonar-upgrader/variables.tf b/modules/aws/sonar-upgrader/variables.tf index f73e99841..a2a013f34 100644 --- a/modules/aws/sonar-upgrader/variables.tf +++ b/modules/aws/sonar-upgrader/variables.tf @@ -110,6 +110,16 @@ variable "run_postflight_validations" { description = "Whether to run the postflight validations or skip them" } +variable "ignore_healthcheck_warnings" { + type = bool + default = false +} + +variable "ignore_healthcheck_checks" { + type = list(string) + default = [] +} + #variable "clean_old_deployments" { # type = bool # default = false From fb4b172ce9596e963eaaad418ae9a58e7ae52650 Mon Sep 17 00:00:00 2001 From: Christopher Papke Date: Wed, 28 Feb 2024 11:08:24 -0800 Subject: [PATCH 2/4] fix tests --- .../python_upgrader/tests/test_main.py | 15 ++++++++++++--- .../python_upgrader/upgrade/main.py | 10 +++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py b/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py index 82cabe3d7..b48308be6 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py @@ -65,7 +65,7 @@ def upgrade_status_service_mock(mocker): upgrade_status_service_mock = mocker.Mock() mocker.patch('upgrade.main.UpgradeStatusService', return_value=upgrade_status_service_mock) mocker.patch.object(upgrade_status_service_mock, 'should_test_connection', return_value=True) - mocker.patch.object(upgrade_status_service_mock, 'should_collect_python_location', return_value=True) + mocker.patch.object(upgrade_status_service_mock, 'should_collect_node_info', return_value=True) mocker.patch.object(upgrade_status_service_mock, 'should_run_preflight_validations', return_value=True) mocker.patch.object(upgrade_status_service_mock, 'should_run_upgrade', return_value=True) mocker.patch.object(upgrade_status_service_mock, 'should_run_postflight_validations', return_value=True) @@ -88,7 +88,10 @@ def run_remote_script_mock(mocker): @pytest.fixture def collect_node_info_mock(mocker): - yield mocker.patch('upgrade.main.collect_node_info') + def set_python_version(extended_node): + extended_node['sysconfig']['JSONAR_VERSION'] = '4.14.0.10.0' + + return mocker.patch('upgrade.main.collect_node_info', side_effect=set_python_version) @pytest.fixture(autouse=True) @@ -300,7 +303,13 @@ def test_main_node_info_failure_with_stop_on_failure_false( # given setup_custom_args(args, [{"main": gw1}, {"main": gw2}], [{"main": hub1}], True, True, True, True, False) # fail if host is host1 - collect_node_info_mock.side_effect = (lambda en: not en.get('dsf_node').get('host') == 'host1') + def collect_node_info(en): + if en.get('dsf_node').get('host') == 'host1': + raise Exception() + else: + en['sysconfig']['JSONAR_VERSION'] = '4.14.0.10.0' + + collect_node_info_mock.side_effect = collect_node_info mocker.patch.object(upgrade_status_service_mock, 'should_run_preflight_validations', side_effect=lambda host: host != "host1") mocker.patch.object(upgrade_status_service_mock, 'should_run_upgrade', side_effect=lambda host: host != "host1") mocker.patch.object(upgrade_status_service_mock, 'should_run_postflight_validations', side_effect=lambda host: host != "host1") diff --git a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py index 9df595f07..36ffd05eb 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py @@ -21,6 +21,7 @@ SONAR_INSTALLATION_S3_PREFIX = "sonar" UNDEFINED_PYTHON_LOCATION = "UNDEFINED_PYTHON_LOCATION" +UNDEFINED_SYSCONFIG = "UNDEFINED_SYSCONFIG" # Globals _connection_timeout = None @@ -149,7 +150,14 @@ def create_extended_node(dsf_node, dsf_node_id, dsf_node_name): "dsf_node": dsf_node, "dsf_node_id": dsf_node_id, "dsf_node_name": dsf_node_name, - "python_location": UNDEFINED_PYTHON_LOCATION # Will be filled later + "python_location": UNDEFINED_PYTHON_LOCATION, # Will be filled later + "sysconfig": { + "JSONAR_BASEDIR": UNDEFINED_SYSCONFIG, + "JSONAR_DATADIR": UNDEFINED_SYSCONFIG, + "JSONAR_LOGDIR": UNDEFINED_SYSCONFIG, + "JSONAR_LOCALDIR": UNDEFINED_SYSCONFIG, + "JSONAR_VERSION": UNDEFINED_SYSCONFIG, + }, } From 301d4ba480c3b0dae6487e487114aa8ea695fb2c Mon Sep 17 00:00:00 2001 From: Christopher Papke Date: Fri, 8 Mar 2024 11:37:36 -0800 Subject: [PATCH 3/4] updates for comments --- .../sonar-upgrader/python_upgrader/README.md | 10 +- .../python_upgrader/tests/test_main.py | 169 +++++++++++++++++- .../python_upgrader/upgrade/main.py | 18 +- modules/aws/sonar-upgrader/variables.tf | 10 +- 4 files changed, 184 insertions(+), 23 deletions(-) diff --git a/modules/aws/sonar-upgrader/python_upgrader/README.md b/modules/aws/sonar-upgrader/python_upgrader/README.md index 1da0f0a75..af4f196c4 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/README.md +++ b/modules/aws/sonar-upgrader/python_upgrader/README.md @@ -65,7 +65,9 @@ instructions to accomplish this task: "dr": { "host": "10.2.1.2", "ssh_user": "ec2-user", - "ssh_private_key_file_path": "/home/ssh_key2.pem" + "ssh_private_key_file_path": "/home/ssh_key2.pem". + "ignore_healthcheck_warnings": true, + "ingore_healthcheck_checks": ["cpu-count"] } } ] @@ -76,7 +78,7 @@ instructions to accomplish this task: For example: ``` - [{"main":{"host":"10.0.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}},{"main":{"host":"10.0.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}}] + [{"main":{"host":"10.0.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}},{"main":{"host":"10.0.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem","ignore_healthcheck_warnings":true,"ingore_healthcheck_checks":["cpu-count"]}}] ``` 3. Wrap the JSON one-liner with single quotes (to avoid collision with the double quotes within the JSON structure) and use it @@ -87,7 +89,7 @@ instructions to accomplish this task: ``` python3 -u -m upgrade.main - --agentless_gws '[{"main":{"host":"10.0.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}},{"main":{"host":"10.0.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}}]' + --agentless_gws '[{"main":{"host":"10.0.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}},{"main":{"host":"10.0.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem","ignore_healthcheck_warnings":true,"ingore_healthcheck_checks":["cpu-count"]}}]' --dsf_hubs '[{"main":{"host":"52.52.52.177","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}}]' --target_version "4.12.0.10.0" ``` @@ -100,7 +102,7 @@ instructions to accomplish this task: ``` python3 -u -m upgrade.main - --agentless_gws '[{"main":{"host":"10.0.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}},{"main":{"host":"10.0.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}}]' + --agentless_gws '[{"main":{"host":"10.0.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.1","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}},{"main":{"host":"10.0.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"},"dr":{"host":"10.2.1.2","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem","ignore_healthcheck_warnings":true,"ingore_healthcheck_checks":["cpu-count"]}}]' --dsf_hubs '[{"main":{"host":"52.52.52.177","ssh_user":"ec2-user","ssh_private_key_file_path":"/home/ssh_key2.pem"}}]' --target_version "4.12.0.10.0" --test_connection "true" diff --git a/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py b/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py index b48308be6..36639ea5e 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/tests/test_main.py @@ -328,6 +328,36 @@ def collect_node_info(en): assert count_remote_calls_with_host_and_script(call_args_list, host, "run_postflight_validations.py") == 1 +def test_main_node_info_failure_with_stop_on_failure_false_4_15( + args, upgrade_status_service_mock, test_connection_mock, run_remote_script_mock, collect_node_info_mock, mocker): + # given + setup_custom_args(args, [{"main": gw1}, {"main": gw2}], [{"main": hub1}], True, True, True, True, False) + # fail if host is host1 + def collect_node_info(en): + if en.get('dsf_node').get('host') == 'host1': + raise Exception() + else: + en['sysconfig']['JSONAR_VERSION'] = '4.15.0.10.0' + + collect_node_info_mock.side_effect = collect_node_info + mocker.patch.object(upgrade_status_service_mock, 'should_run_preflight_validations', side_effect=lambda host: host != "host1") + mocker.patch.object(upgrade_status_service_mock, 'should_run_upgrade', side_effect=lambda host: host != "host1") + mocker.patch.object(upgrade_status_service_mock, 'should_run_postflight_validations', side_effect=lambda host: host != "host1") + + # when + main(args) + + # then + assert test_connection_mock.call_count == 3 + call_args_list = run_remote_script_mock.call_args_list + assert len(call_args_list) == 6 + assert collect_node_info_mock.call_count == 3 + for host in ["host2", "host100"]: + assert count_remote_calls_with_host_and_script(call_args_list, host, "health-checker") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, host, "upgrade_v4_10.sh") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, host, "run_postflight_validations.py") == 1 + + @pytest.mark.parametrize("preflight_not_pass_hosts, preflight_error_hosts", [ (["host1"], []), ([], ["host1"]), @@ -427,6 +457,106 @@ def test_main_hadr_set_skip_node_after_hadr_postflight_failure_stop_on_failure_f assert count_remote_calls_with_host_and_script(call_args_list, "host102", "run_preflight_validations.py") == 1 +@pytest.mark.parametrize("warning_hosts", [ + (["host100"]), + ([]), +]) +def test_healthcheck_ignore_warnings( + args, run_remote_script_mock, warning_hosts, test_connection_mock, + collect_node_info_mock): + setup_custom_args(args, [], [{"main": hub1}], ignore_healthcheck_warnings=True) + run_remote_script_mock.side_effect = create_mocked_run_remote_script_side_effects( + preflight_validations_not_pass_hosts=warning_hosts, + ) + def collect_node_info(en): + en['sysconfig']['JSONAR_VERSION'] = '4.15.0.10.0' + + collect_node_info_mock.side_effect = collect_node_info + + main(args) + + assert test_connection_mock.call_count == 1 + call_args_list = run_remote_script_mock.call_args_list + assert len(call_args_list) == 3 + assert collect_node_info_mock.call_count == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "health-checker") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "upgrade_v4_10.sh") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "run_postflight_validations.py") == 1 + + +def test_healthcheck_fail_warnings( + args, run_remote_script_mock, test_connection_mock, + collect_node_info_mock): + setup_custom_args(args, [], [{"main": hub1}]) + run_remote_script_mock.side_effect = create_mocked_run_remote_script_side_effects( + preflight_validations_not_pass_hosts=["host100"], + ) + def collect_node_info(en): + en['sysconfig']['JSONAR_VERSION'] = '4.15.0.10.0' + + collect_node_info_mock.side_effect = collect_node_info + + main(args) + + assert test_connection_mock.call_count == 1 + call_args_list = run_remote_script_mock.call_args_list + assert len(call_args_list) == 1 + assert collect_node_info_mock.call_count == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "health-checker") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "upgrade_v4_10.sh") == 0 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "run_postflight_validations.py") == 0 + + +@pytest.mark.parametrize("error_hosts", [ + (["host100"]), + ([]), +]) +def test_healthcheck_ignore_failed_check( + args, run_remote_script_mock, error_hosts, test_connection_mock, + collect_node_info_mock): + setup_custom_args(args, [], [{"main": hub1}], ignore_healthcheck_checks=["failed_check"]) + run_remote_script_mock.side_effect = create_mocked_run_remote_script_side_effects( + preflight_validations_error_hosts=error_hosts, + ) + def collect_node_info(en): + en['sysconfig']['JSONAR_VERSION'] = '4.15.0.10.0' + + collect_node_info_mock.side_effect = collect_node_info + + main(args) + + assert test_connection_mock.call_count == 1 + call_args_list = run_remote_script_mock.call_args_list + assert len(call_args_list) == 3 + assert collect_node_info_mock.call_count == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "health-checker") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "upgrade_v4_10.sh") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "run_postflight_validations.py") == 1 + + +def test_healthcheck_fail_failed_check( + args, run_remote_script_mock, test_connection_mock, + collect_node_info_mock): + setup_custom_args(args, [], [{"main": hub1}]) + run_remote_script_mock.side_effect = create_mocked_run_remote_script_side_effects( + preflight_validations_error_hosts=["host100"], + ) + def collect_node_info(en): + en['sysconfig']['JSONAR_VERSION'] = '4.15.0.10.0' + + collect_node_info_mock.side_effect = collect_node_info + + main(args) + + assert test_connection_mock.call_count == 1 + call_args_list = run_remote_script_mock.call_args_list + assert len(call_args_list) == 1 + assert collect_node_info_mock.call_count == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "health-checker") == 1 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "upgrade_v4_10.sh") == 0 + assert count_remote_calls_with_host_and_script(call_args_list, "host100", "run_postflight_validations.py") == 0 + + @pytest.mark.xfail(raises=UpgradeException) def test_main_raise_exception_on_overall_status_failed( args, upgrade_status_service_mock, test_connection_mock, run_remote_script_mock, collect_node_info_mock, mocker): @@ -445,17 +575,30 @@ def test_main_raise_exception_on_overall_status_failed( assert collect_node_info_mock.call_count == 1 -def setup_custom_args(args, agentless_gws, dsf_hubs, test_connection, run_preflight_validations, run_upgrade, - run_postflight_validations, stop_on_failure, tarball_location=None): - args.agentless_gws = json.dumps(agentless_gws) - args.dsf_hubs = json.dumps(dsf_hubs) - args.test_connection = test_connection - args.run_preflight_validations = run_preflight_validations - args.run_upgrade = run_upgrade - args.run_postflight_validations = run_postflight_validations - args.stop_on_failure = stop_on_failure +def setup_custom_args(args, agentless_gws=None, dsf_hubs=None, test_connection=None, + run_preflight_validations=None, run_upgrade=None, + run_postflight_validations=None, stop_on_failure=None, tarball_location=None, + ignore_healthcheck_warnings=None, ignore_healthcheck_checks=None): + if agentless_gws is not None: + args.agentless_gws = json.dumps(agentless_gws) + if dsf_hubs is not None: + args.dsf_hubs = json.dumps(dsf_hubs) + if test_connection is not None: + args.test_connection = test_connection + if run_preflight_validations is not None: + args.run_preflight_validations = run_preflight_validations + if run_upgrade is not None: + args.run_upgrade = run_upgrade + if run_postflight_validations is not None: + args.run_postflight_validations = run_postflight_validations + if stop_on_failure is not None: + args.stop_on_failure = stop_on_failure if tarball_location is not None: args.tarball_location = tarball_location + if ignore_healthcheck_warnings is not None: + args.ignore_healthcheck_warnings = ignore_healthcheck_warnings + if ignore_healthcheck_checks is not None: + args.ignore_healthcheck_check = ignore_healthcheck_checks def create_mocked_run_remote_script_side_effects(preflight_validations_not_pass_hosts=None, @@ -474,6 +617,14 @@ def mocked_run_remote_script(dsf_node, script_contents, script_run_command): else: return 'Preflight validations result: {"higher_target_version": true, "min_version": true, ' \ '"max_version_hop": true, "enough_free_disk_space": true}' + elif "health-checker" in script_run_command: + if preflight_validations_error_hosts is not None and host in preflight_validations_error_hosts: + return json.dumps({'machines': [{'results': [{'status': 'FAILURE', 'name': 'failed_check'}]}]}) + elif preflight_validations_not_pass_hosts is not None and host in preflight_validations_not_pass_hosts: + return json.dumps({'machines': [{'results': [{'status': 'WARNING', 'name': 'failed_check'}]}]}) + else: + return json.dumps({'machines': [{'results': []}]}) + elif "upgrade_v4_10.sh" in script_contents: if upgrade_error_hosts is not None and host in upgrade_error_hosts: return "upgrade error" diff --git a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py index 36ffd05eb..24fbb4ed0 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py @@ -424,12 +424,7 @@ def run_preflight_validations_for_node( version_list = extended_node['sysconfig']['JSONAR_VERSION'].split('-')[0].split('.') version_tuple = tuple(map(int, version_list)) if version_tuple >= (4, 15): - script_output = run_remote_script(extended_node['dsf_node'], "", f"sudo {extended_node['sysconfig']['JSONAR_BASEDIR']}/bin/health-checker --quiet --output JSON --target-version {target_version}") - results = list(chain.from_iterable(machine['results'] for machine in json.loads(script_output)['machines'])) - results = [result for result in results if result['status'] in ['WARNING', 'FAILURE']] - if ignore_healthcheck_warning: - results = [result for result in results if result['status'] != 'WARNING'] - results = [result for result in results if result['name'] not in ignore_healthcheck_checks] + results = get_healthchecker_results(extended_node, ignore_healthcheck_warning, ignore_healthcheck_checks, target_version) if results: error_message = f'Healthchecks failed: {results}' else: @@ -462,6 +457,17 @@ def run_preflight_validations_for_node( return True +def get_healthchecker_results(extended_node, ignore_healthcheck_warning, ignore_healthcheck_checks, target_version): + healthcheck_script = f"sudo {extended_node['sysconfig']['JSONAR_BASEDIR']}/bin/health-checker --quiet --output JSON --target-version {target_version}" + script_output = run_remote_script(extended_node['dsf_node'], healthcheck_script, healthcheck_script) + results = list(chain.from_iterable(machine['results'] for machine in json.loads(script_output)['machines'])) + results = [result for result in results if result['status'] in ['WARNING', 'FAILURE']] + if ignore_healthcheck_warning: + results = [result for result in results if result['status'] != 'WARNING'] + results = [result for result in results if result['name'] not in ignore_healthcheck_checks] + return results + + def run_preflight_validations_script(target_version, dsf_node, dsf_node_name, python_location): print(f"Running preflight validations for {dsf_node_name}") script_file_path = build_script_file_path(PREFLIGHT_VALIDATIONS_SCRIPT_NAME) diff --git a/modules/aws/sonar-upgrader/variables.tf b/modules/aws/sonar-upgrader/variables.tf index a2a013f34..cc2bec9dc 100644 --- a/modules/aws/sonar-upgrader/variables.tf +++ b/modules/aws/sonar-upgrader/variables.tf @@ -111,13 +111,15 @@ variable "run_postflight_validations" { } variable "ignore_healthcheck_warnings" { - type = bool - default = false + type = bool + default = false + description = "If true, we ignore any warnings from the healthchecker" } variable "ignore_healthcheck_checks" { - type = list(string) - default = [] + type = list(string) + default = [] + description = "We will ignore any failed healthchecks listed here when running the preflight check" } #variable "clean_old_deployments" { From cfff272d3c46ffcb034604b52f5f8b1418741e79 Mon Sep 17 00:00:00 2001 From: Christopher Papke Date: Mon, 11 Mar 2024 09:32:16 -0700 Subject: [PATCH 4/4] added print statement --- modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py index 24fbb4ed0..9dfbb96c9 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py @@ -426,6 +426,7 @@ def run_preflight_validations_for_node( if version_tuple >= (4, 15): results = get_healthchecker_results(extended_node, ignore_healthcheck_warning, ignore_healthcheck_checks, target_version) if results: + print(f"### Preflight validations didn't pass for {extended_node.get('dsf_node_name')}") error_message = f'Healthchecks failed: {results}' else: preflight_validations_result = run_preflight_validations_script(