From cdd76f47d8549c645a43f96f9817437e32b81e7a Mon Sep 17 00:00:00 2001 From: Vadym Yarosh Date: Sun, 15 Oct 2023 12:08:46 +0200 Subject: [PATCH] PMM-7 migrated PMM container components tests to unittest --- pmm-tests/check_upgrade.py | 277 +++++++++++++++++++++++-------------- 1 file changed, 172 insertions(+), 105 deletions(-) diff --git a/pmm-tests/check_upgrade.py b/pmm-tests/check_upgrade.py index 5377c36f..4bd7ac71 100644 --- a/pmm-tests/check_upgrade.py +++ b/pmm-tests/check_upgrade.py @@ -1,7 +1,14 @@ #!/usr/bin/env python import argparse -import subprocess import os +import subprocess +import sys +import unittest + + +POST_UPGRADE = 'Post-upgrade test' +RUNNING = 'RUNNING' +pmm_server_docker_container = '' def parse_args(): @@ -10,111 +17,171 @@ def parse_args(): parser.add_argument('-p', '--pre_post', help="Pass 'pre' from pre-upgrade stage and pass 'post' for post-upgrade " "stage", default='post') parser.add_argument('-e', '--env', help="Pass 'ami' if it is AMI env, otherwise leave empty", default='not_ami') - - args = parser.parse_args() - - return args + return parser.parse_args() def verify_command(command): - with open("/tmp/output.log", "a") as output: - error_code = subprocess.call(command, shell=True, stdout=output, stderr=output) - if error_code != 0: - output = subprocess.getoutput(command.split("|")[0]) - assert error_code == 0, f"expected to run {command} without errors. \nOutput: {output}" - - -def main(): + from tempfile import TemporaryFile + # with open("/tmp/output.log", "a") as output: + with TemporaryFile() as output: + try: + return subprocess.check_output(command, shell=True, text=True, stderr=output).rstrip() + except subprocess.CalledProcessError as e: + assert e.returncode == 0, f"'{command}' exited with {e.returncode} {output.read()}" + + +class PmmServerComponents(unittest.TestCase): + + def test_percona_qan_api2_version(self): + out = grep_rpm('percona-qan-api2-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_percona_dashboards_version(self): + out = grep_rpm('percona-dashboards-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_clickhouse_version(self): + """PMM-12223 - Verify Clickhouse is v23.8 or later since 2.41.0""" + if pmm_minor_version < 41: self.skipTest('Since version 2.41.0') + if is_ami: + out = verify_command('sudo clickhouse local --version') + else: + out = verify_command(f"docker exec {get_container_name()} clickhouse local --version") + self.assertIn('23.8.2.7', out, 'Unexpected version!') + + def test_pmm_update_version(self): + if pmm_version != "2.25.0": self.skipTest('for 2.25.0 only!') + out = grep_rpm('pmm-update-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_pmm_managed_version(self): + out = grep_rpm('pmm-managed-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_pmm2_client_version(self): + out = grep_rpm('pmm2-client-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_dbaas_controller_version(self): + if test_mode != "post": self.skipTest(POST_UPGRADE) + out = grep_rpm('dbaas-controller-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_pmm_dump_version(self): + if test_mode != "post": self.skipTest(POST_UPGRADE) + out = grep_rpm('pmm-dump-') + self.assertIn(pmm_version, out, 'Unexpected version!') + + def test_qan_api2_status(self): + out = grep_supervisor_status('qan-api2') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_alert_manager_status(self): + out = grep_supervisor_status('alertmanager') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_clickhouse_status(self): + out = grep_supervisor_status('clickhouse') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_grafana_status(self): + out = grep_supervisor_status('grafana') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_nginx_status(self): + out = grep_supervisor_status('nginx') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_pmm_agent_status(self): + out = grep_supervisor_status('pmm-agent') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_pmm_managed_status(self): + out = grep_supervisor_status('pmm-managed') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_postgresql_status(self): + out = grep_supervisor_status('postgresql') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_victoriametrics_status(self): + if test_mode != "post": self.skipTest(POST_UPGRADE) + out = grep_supervisor_status('victoriametrics') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_victoria_metrics_alert_status(self): + if test_mode != "post": self.skipTest(POST_UPGRADE) + out = grep_supervisor_status('vmalert') + self.assertIn(RUNNING, out, 'Status is Not running!') + + def test_victoria_metrics_version(self): + if test_mode != "post": self.skipTest(POST_UPGRADE) + if is_ami: + out = verify_command('sudo victoriametrics --version').strip() + else: + out = verify_command(f"docker exec {get_container_name()} victoriametrics --version").strip() + self.assertIn('v1.93.4', out, 'Unexpected version!') + + def test_vertamedia_clickhouse_plugin_absent(self): + # """PMM-T1758 - Verify vertamedia-clickhouse-datasource plugin is not installed after upgrade to 2.38.0""" + if pmm_minor_version < 38: self.skipTest('Since version 2.41.0') + if is_ami: + out = verify_command(f"{grafana_cli} plugins ls") + else: + out = verify_command( + f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ " + f"{get_container_name()} {grafana_cli} plugins ls") + self.assertNotIn('vertamedia', out, 'Must be absent!') + + # Disabled tests AMI+Docker: + # verify_command('grafana cli plugins ls | grep alexanderzobnin-zabbix-app') + # if (do_docker_way == "yes" and pmm_minor_v > 22) or (do_docker_way != "yes"): + # verify_command( + # f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} " + # f"{grafana_cli} plugins ls | grep alexanderzobnin-zabbix-app") + + # verify_command('grafana cli plugins ls | grep "vertamedia-clickhouse-datasource @ 2.4.4"') + # verify_command( + # f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} " + # f"{grafana_cli} plugins ls | grep \"vertamedia-clickhouse-datasource @ 2.4.4\"") + + +if __name__ == '__main__': args = parse_args() - - if args.env in "ami": - verify_command('rpm -qa | grep percona-qan-api2-' + args.version) - verify_command('rpm -qa | grep percona-dashboards-' + args.version) - if args.version == "2.25.0": - verify_command('rpm -qa | grep pmm-update-' + args.version) - - verify_command('rpm -qa | grep pmm-managed-' + args.version) - verify_command('rpm -qa | grep pmm2-client-' + args.version) - verify_command('sudo supervisorctl status | grep qan-api2 | grep RUNNING') - verify_command('sudo supervisorctl status | grep alertmanager | grep RUNNING') - verify_command('sudo supervisorctl status | grep clickhouse | grep RUNNING') - verify_command('sudo supervisorctl status | grep grafana | grep RUNNING') - verify_command('sudo supervisorctl status | grep nginx | grep RUNNING') - verify_command('sudo supervisorctl status | grep pmm-agent | grep RUNNING') - verify_command('sudo supervisorctl status | grep pmm-managed | grep RUNNING') - verify_command('sudo supervisorctl status | grep postgresql | grep RUNNING') - - if args.pre_post == "post": - verify_command('rpm -qa | grep dbaas-controller-' + args.version) - verify_command('rpm -qa | grep pmm-dump-' + args.version) - verify_command('sudo supervisorctl status | grep victoriametrics | grep RUNNING') - verify_command('sudo supervisorctl status | grep vmalert | grep RUNNING') - # verify_command('grafana cli plugins ls | grep "vertamedia-clickhouse-datasource @ 2.4.4"') - # verify_command('grafana cli plugins ls | grep alexanderzobnin-zabbix-app') - verify_command('sudo victoriametrics --version | grep v1.93.4') - else: - pmm_server_docker_container = subprocess.getoutput("docker ps --format \"table {{.ID}}\t{{.Image}}\t{{" - ".Names}}\" | grep 'pmm-server' | awk '{print $3}'") - print(f"PMM Server container name is {pmm_server_docker_container}, verification for {args.pre_post}") - - verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep percona-qan-api2-{args.version}") - verify_command( - f"docker exec {pmm_server_docker_container} rpm -qa | grep percona-dashboards-{args.version}") - if args.version != "2.25.0": - verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm-update-{args.version}") - verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm-managed-{args.version}") - verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm2-client-{args.version}") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep qan-api2 | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep alertmanager | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep clickhouse | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep grafana | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep nginx | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep pmm-agent | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep pmm-managed | grep " - f"RUNNING") - verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep postgresql | grep " - f"RUNNING") - if args.pre_post == "post": - verify_command( - f"docker exec {pmm_server_docker_container} rpm -qa | grep dbaas-controller-{args.version}") - verify_command( - f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm-dump-{args.version}") - verify_command( - f"docker exec {pmm_server_docker_container} supervisorctl status | grep victoriametrics | grep " - f"RUNNING") - verify_command( - f"docker exec {pmm_server_docker_container} supervisorctl status | grep vmalert | grep " - f"RUNNING") - verify_command( - f"docker exec {pmm_server_docker_container} victoriametrics --version | grep v1.93.4") - - docker_version = os.getenv("DOCKER_VERSION") - do_docker_way = os.getenv("PERFORM_DOCKER_WAY_UPGRADE") - pmm_minor_v = int(args.version.split('.')[1]) - grafana_cli = "grafana cli" if pmm_minor_v >= 39 else "grafana-cli" - - ### PMM-T1758 - Verify vertamedia-clickhouse-datasource plugin is not installed after upgrade to 2.38.0 - if pmm_minor_v >= 38: - verify_command( - f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ " - f"{pmm_server_docker_container} {grafana_cli} plugins ls | exit $(grep -c vertamedia)") - - # if (do_docker_way == "yes" and pmm_minor_v > 22) or (do_docker_way != "yes"): - # verify_command( - # f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} " - # f"{grafana_cli} plugins ls | grep alexanderzobnin-zabbix-app") - - # verify_command( - # f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} " - # f"{grafana_cli} plugins ls | grep \"vertamedia-clickhouse-datasource @ 2.4.4\"") - print(f"Post upgrade verification complete! for {args.version}") - - -if __name__ == "__main__": - main() + pmm_version = args.version + pmm_minor_version = int(args.version.split('.')[1]) + test_mode = args.pre_post + is_ami = args.env in "ami" + do_docker_way = os.getenv("PERFORM_DOCKER_WAY_UPGRADE") + grafana_cli = "grafana cli" if pmm_minor_version >= 39 else "grafana-cli" + + + def get_container_name(): + """Lazi initialization""" + global pmm_server_docker_container + if not pmm_server_docker_container: + pmm_server_docker_container = verify_command( + """docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" | grep 'pmm-server' | awk '{print $3}'""" + ).strip() + return pmm_server_docker_container + + + def grep_rpm(query): + """Polymorphic shortcut to use in test""" + if is_ami: + return verify_command(f"rpm -qa | grep {query}") + else: + return verify_command(f"docker exec {get_container_name()} rpm -qa | grep {query}") + + + def grep_supervisor_status(name): + """Polymorphic shortcut to use in test""" + if is_ami: + return verify_command(f"sudo supervisorctl status | grep {name}") + else: + return verify_command(f"docker exec {get_container_name()} supervisorctl status | grep {name}") + + + # leaving sys.argv[0] alone + del sys.argv[1:] + unittest.main(verbosity=2)