Skip to content

Commit

Permalink
PMM-7 migrated PMM container components tests to unittest
Browse files Browse the repository at this point in the history
  • Loading branch information
Vadym Yarosh committed Oct 15, 2023
1 parent 8bb4a76 commit cdd76f4
Showing 1 changed file with 172 additions and 105 deletions.
277 changes: 172 additions & 105 deletions pmm-tests/check_upgrade.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
#!/usr/bin/env python
import argparse
import subprocess
import os
import subprocess
import sys
import unittest


POST_UPGRADE = 'Post-upgrade test'
RUNNING = 'RUNNING'
pmm_server_docker_container = ''


def parse_args():
Expand All @@ -10,111 +17,171 @@ def parse_args():
parser.add_argument('-p', '--pre_post', help="Pass 'pre' from pre-upgrade stage and pass 'post' for post-upgrade "
"stage", default='post')
parser.add_argument('-e', '--env', help="Pass 'ami' if it is AMI env, otherwise leave empty", default='not_ami')

args = parser.parse_args()

return args
return parser.parse_args()


def verify_command(command):
with open("/tmp/output.log", "a") as output:
error_code = subprocess.call(command, shell=True, stdout=output, stderr=output)
if error_code != 0:
output = subprocess.getoutput(command.split("|")[0])
assert error_code == 0, f"expected to run {command} without errors. \nOutput: {output}"


def main():
from tempfile import TemporaryFile
# with open("/tmp/output.log", "a") as output:
with TemporaryFile() as output:
try:
return subprocess.check_output(command, shell=True, text=True, stderr=output).rstrip()
except subprocess.CalledProcessError as e:
assert e.returncode == 0, f"'{command}' exited with {e.returncode} {output.read()}"


class PmmServerComponents(unittest.TestCase):

def test_percona_qan_api2_version(self):
out = grep_rpm('percona-qan-api2-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_percona_dashboards_version(self):
out = grep_rpm('percona-dashboards-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_clickhouse_version(self):
"""PMM-12223 - Verify Clickhouse is v23.8 or later since 2.41.0"""
if pmm_minor_version < 41: self.skipTest('Since version 2.41.0')
if is_ami:
out = verify_command('sudo clickhouse local --version')
else:
out = verify_command(f"docker exec {get_container_name()} clickhouse local --version")
self.assertIn('23.8.2.7', out, 'Unexpected version!')

def test_pmm_update_version(self):
if pmm_version != "2.25.0": self.skipTest('for 2.25.0 only!')
out = grep_rpm('pmm-update-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_pmm_managed_version(self):
out = grep_rpm('pmm-managed-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_pmm2_client_version(self):
out = grep_rpm('pmm2-client-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_dbaas_controller_version(self):
if test_mode != "post": self.skipTest(POST_UPGRADE)
out = grep_rpm('dbaas-controller-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_pmm_dump_version(self):
if test_mode != "post": self.skipTest(POST_UPGRADE)
out = grep_rpm('pmm-dump-')
self.assertIn(pmm_version, out, 'Unexpected version!')

def test_qan_api2_status(self):
out = grep_supervisor_status('qan-api2')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_alert_manager_status(self):
out = grep_supervisor_status('alertmanager')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_clickhouse_status(self):
out = grep_supervisor_status('clickhouse')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_grafana_status(self):
out = grep_supervisor_status('grafana')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_nginx_status(self):
out = grep_supervisor_status('nginx')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_pmm_agent_status(self):
out = grep_supervisor_status('pmm-agent')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_pmm_managed_status(self):
out = grep_supervisor_status('pmm-managed')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_postgresql_status(self):
out = grep_supervisor_status('postgresql')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_victoriametrics_status(self):
if test_mode != "post": self.skipTest(POST_UPGRADE)
out = grep_supervisor_status('victoriametrics')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_victoria_metrics_alert_status(self):
if test_mode != "post": self.skipTest(POST_UPGRADE)
out = grep_supervisor_status('vmalert')
self.assertIn(RUNNING, out, 'Status is Not running!')

def test_victoria_metrics_version(self):
if test_mode != "post": self.skipTest(POST_UPGRADE)
if is_ami:
out = verify_command('sudo victoriametrics --version').strip()
else:
out = verify_command(f"docker exec {get_container_name()} victoriametrics --version").strip()
self.assertIn('v1.93.4', out, 'Unexpected version!')

def test_vertamedia_clickhouse_plugin_absent(self):
# """PMM-T1758 - Verify vertamedia-clickhouse-datasource plugin is not installed after upgrade to 2.38.0"""
if pmm_minor_version < 38: self.skipTest('Since version 2.41.0')
if is_ami:
out = verify_command(f"{grafana_cli} plugins ls")
else:
out = verify_command(
f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ "
f"{get_container_name()} {grafana_cli} plugins ls")
self.assertNotIn('vertamedia', out, 'Must be absent!')

# Disabled tests AMI+Docker:
# verify_command('grafana cli plugins ls | grep alexanderzobnin-zabbix-app')
# if (do_docker_way == "yes" and pmm_minor_v > 22) or (do_docker_way != "yes"):
# verify_command(
# f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} "
# f"{grafana_cli} plugins ls | grep alexanderzobnin-zabbix-app")

# verify_command('grafana cli plugins ls | grep "vertamedia-clickhouse-datasource @ 2.4.4"')
# verify_command(
# f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} "
# f"{grafana_cli} plugins ls | grep \"vertamedia-clickhouse-datasource @ 2.4.4\"")


if __name__ == '__main__':
args = parse_args()

if args.env in "ami":
verify_command('rpm -qa | grep percona-qan-api2-' + args.version)
verify_command('rpm -qa | grep percona-dashboards-' + args.version)
if args.version == "2.25.0":
verify_command('rpm -qa | grep pmm-update-' + args.version)

verify_command('rpm -qa | grep pmm-managed-' + args.version)
verify_command('rpm -qa | grep pmm2-client-' + args.version)
verify_command('sudo supervisorctl status | grep qan-api2 | grep RUNNING')
verify_command('sudo supervisorctl status | grep alertmanager | grep RUNNING')
verify_command('sudo supervisorctl status | grep clickhouse | grep RUNNING')
verify_command('sudo supervisorctl status | grep grafana | grep RUNNING')
verify_command('sudo supervisorctl status | grep nginx | grep RUNNING')
verify_command('sudo supervisorctl status | grep pmm-agent | grep RUNNING')
verify_command('sudo supervisorctl status | grep pmm-managed | grep RUNNING')
verify_command('sudo supervisorctl status | grep postgresql | grep RUNNING')

if args.pre_post == "post":
verify_command('rpm -qa | grep dbaas-controller-' + args.version)
verify_command('rpm -qa | grep pmm-dump-' + args.version)
verify_command('sudo supervisorctl status | grep victoriametrics | grep RUNNING')
verify_command('sudo supervisorctl status | grep vmalert | grep RUNNING')
# verify_command('grafana cli plugins ls | grep "vertamedia-clickhouse-datasource @ 2.4.4"')
# verify_command('grafana cli plugins ls | grep alexanderzobnin-zabbix-app')
verify_command('sudo victoriametrics --version | grep v1.93.4')
else:
pmm_server_docker_container = subprocess.getoutput("docker ps --format \"table {{.ID}}\t{{.Image}}\t{{"
".Names}}\" | grep 'pmm-server' | awk '{print $3}'")
print(f"PMM Server container name is {pmm_server_docker_container}, verification for {args.pre_post}")

verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep percona-qan-api2-{args.version}")
verify_command(
f"docker exec {pmm_server_docker_container} rpm -qa | grep percona-dashboards-{args.version}")
if args.version != "2.25.0":
verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm-update-{args.version}")
verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm-managed-{args.version}")
verify_command(f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm2-client-{args.version}")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep qan-api2 | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep alertmanager | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep clickhouse | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep grafana | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep nginx | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep pmm-agent | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep pmm-managed | grep "
f"RUNNING")
verify_command(f"docker exec {pmm_server_docker_container} supervisorctl status | grep postgresql | grep "
f"RUNNING")
if args.pre_post == "post":
verify_command(
f"docker exec {pmm_server_docker_container} rpm -qa | grep dbaas-controller-{args.version}")
verify_command(
f"docker exec {pmm_server_docker_container} rpm -qa | grep pmm-dump-{args.version}")
verify_command(
f"docker exec {pmm_server_docker_container} supervisorctl status | grep victoriametrics | grep "
f"RUNNING")
verify_command(
f"docker exec {pmm_server_docker_container} supervisorctl status | grep vmalert | grep "
f"RUNNING")
verify_command(
f"docker exec {pmm_server_docker_container} victoriametrics --version | grep v1.93.4")

docker_version = os.getenv("DOCKER_VERSION")
do_docker_way = os.getenv("PERFORM_DOCKER_WAY_UPGRADE")
pmm_minor_v = int(args.version.split('.')[1])
grafana_cli = "grafana cli" if pmm_minor_v >= 39 else "grafana-cli"

### PMM-T1758 - Verify vertamedia-clickhouse-datasource plugin is not installed after upgrade to 2.38.0
if pmm_minor_v >= 38:
verify_command(
f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ "
f"{pmm_server_docker_container} {grafana_cli} plugins ls | exit $(grep -c vertamedia)")

# if (do_docker_way == "yes" and pmm_minor_v > 22) or (do_docker_way != "yes"):
# verify_command(
# f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} "
# f"{grafana_cli} plugins ls | grep alexanderzobnin-zabbix-app")

# verify_command(
# f"docker exec -e GF_PLUGIN_DIR=/srv/grafana/plugins/ {pmm_server_docker_container} "
# f"{grafana_cli} plugins ls | grep \"vertamedia-clickhouse-datasource @ 2.4.4\"")
print(f"Post upgrade verification complete! for {args.version}")


if __name__ == "__main__":
main()
pmm_version = args.version
pmm_minor_version = int(args.version.split('.')[1])
test_mode = args.pre_post
is_ami = args.env in "ami"
do_docker_way = os.getenv("PERFORM_DOCKER_WAY_UPGRADE")
grafana_cli = "grafana cli" if pmm_minor_version >= 39 else "grafana-cli"


def get_container_name():
"""Lazi initialization"""
global pmm_server_docker_container
if not pmm_server_docker_container:
pmm_server_docker_container = verify_command(
"""docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" | grep 'pmm-server' | awk '{print $3}'"""
).strip()
return pmm_server_docker_container


def grep_rpm(query):
"""Polymorphic shortcut to use in test"""
if is_ami:
return verify_command(f"rpm -qa | grep {query}")
else:
return verify_command(f"docker exec {get_container_name()} rpm -qa | grep {query}")


def grep_supervisor_status(name):
"""Polymorphic shortcut to use in test"""
if is_ami:
return verify_command(f"sudo supervisorctl status | grep {name}")
else:
return verify_command(f"docker exec {get_container_name()} supervisorctl status | grep {name}")


# leaving sys.argv[0] alone
del sys.argv[1:]
unittest.main(verbosity=2)

0 comments on commit cdd76f4

Please sign in to comment.