From 71bd97475a823f56e328ec981e8533ebc80a509c Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Tue, 16 Jul 2024 14:02:55 +0300 Subject: [PATCH 1/7] PBM-1043 Oplog slicer should survive if either mongo or pbm-agent is unavailable --- pbm-functional/pytest/Dockerfile | 1 + pbm-functional/pytest/conf/pbm/pbm-1043.yaml | 17 ++++ pbm-functional/pytest/test_PBM-1043.py | 92 ++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 pbm-functional/pytest/conf/pbm/pbm-1043.yaml create mode 100644 pbm-functional/pytest/test_PBM-1043.py diff --git a/pbm-functional/pytest/Dockerfile b/pbm-functional/pytest/Dockerfile index 3c449c6c..3f8d8b7a 100644 --- a/pbm-functional/pytest/Dockerfile +++ b/pbm-functional/pytest/Dockerfile @@ -56,6 +56,7 @@ COPY --from=pykmip /root/easy-rsa/easyrsa3/pki/ca.crt /etc/pykmip/ca.crt COPY --from=x509 /etc/x509/ /etc/x509/ COPY conf/supervisord.d/ /etc/supervisord.d/ COPY conf/pbm/minio.yaml /etc/pbm.conf +COPY conf/pbm/pbm-1043.yaml /etc/pbm-1043.conf COPY conf/krb5.conf /etc/krb5.conf COPY keyfile /etc/keyfile COPY mongodb-keyfile /etc/mongodb-keyfile diff --git a/pbm-functional/pytest/conf/pbm/pbm-1043.yaml b/pbm-functional/pytest/conf/pbm/pbm-1043.yaml new file mode 100644 index 00000000..6a0c36c1 --- /dev/null +++ b/pbm-functional/pytest/conf/pbm/pbm-1043.yaml @@ -0,0 +1,17 @@ +storage: + type: s3 + s3: + endpointUrl: http://minio:9000 + bucket: bcp + prefix: pbme2etest + credentials: + access-key-id: "minio1234" + secret-access-key: "minio1234" +pitr: + enabled: true + oplogSpanMin: 0.5 + priority: + rs103:27017: 2 + rs203:27017: 2 + rscfg03:27017: 2 + compression: none diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py new file mode 100644 index 00000000..d803301c --- /dev/null +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -0,0 +1,92 @@ +import pytest +import pymongo +import bson +import testinfra +import time +import os +import docker +import threading + +from datetime import datetime +from cluster import Cluster + +@pytest.fixture(scope="package") +def docker_client(): + return docker.from_env() + +@pytest.fixture(scope="package") +def config(): + return { "mongos": "mongos", + "configserver": + {"_id": "rscfg", "members": [{"host":"rscfg01"},{"host": "rscfg02"},{"host": "rscfg03" }]}, + "shards":[ + {"_id": "rs1", "members": [{"host":"rs101"},{"host": "rs102"},{"host": "rs103"}]}, + {"_id": "rs2", "members": [{"host":"rs201"},{"host": "rs202"},{"host": "rs203"}]} + ]} + +@pytest.fixture(scope="package") +def cluster(config): + return Cluster(config) + +@pytest.fixture(scope="function") +def start_cluster(cluster,request): + try: + cluster.destroy() + os.chmod("/backups",0o777) + os.system("rm -rf /backups/*") + cluster.create() + cluster.setup_pbm() + client=pymongo.MongoClient(cluster.connection) + client.admin.command("enableSharding", "test") + client.admin.command("shardCollection", "test.test", key={"_id": "hashed"}) + yield True + + finally: + if request.config.getoption("--verbose"): + cluster.get_logs() + cluster.destroy(cleanup_backups=True) + +@pytest.mark.timeout(600,func_only=True) +def test_logical_PBM_T255(start_cluster,cluster): + def insert_docs(): + client=pymongo.MongoClient(cluster.connection) + for i in range(200): + client['test']['test'].insert_one({"doc":i}) + time.sleep(1) + + cluster.make_backup("logical") + cluster.exec_pbm_cli("config --file=/etc/pbm-1043.conf") + time.sleep(30) + Cluster.log("Check if PITR is running") + assert cluster.check_pitr() + + Cluster.log("Start inserting docs in the background") + background_insert = threading.Thread(target=insert_docs) + background_insert.start() + time.sleep(60) + + nrs103=testinfra.get_host("docker://rs103") + nrs203=testinfra.get_host("docker://rs203") + + Cluster.log("Oplog slicer should survive if either mongo or pbm-agent is unavailable") + nrs103.check_output('supervisorctl stop mongod') + nrs203.check_output('supervisorctl stop pbm-agent') + + time.sleep(60) + assert cluster.check_pitr() + + Cluster.log("Start pbm-agent and mongod") + nrs103.check_output('supervisorctl start mongod') + nrs203.check_output('supervisorctl start pbm-agent') + + time.sleep(60) + background_insert.join() + assert cluster.check_pitr() + + assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 + pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") + backup="--time=" + pitr + Cluster.log("Time for PITR is: " + pitr) + time.sleep(30) + cluster.make_restore(backup,check_pbm_status=True) + assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 From b633322b3dc7d77641d30aa5608d40e5f148e2e1 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Tue, 16 Jul 2024 14:19:47 +0300 Subject: [PATCH 2/7] PBM-1043 Increase timeout before pitr restore --- pbm-functional/pytest/test_PBM-1043.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py index d803301c..3e69e52d 100644 --- a/pbm-functional/pytest/test_PBM-1043.py +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -87,6 +87,6 @@ def insert_docs(): pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") backup="--time=" + pitr Cluster.log("Time for PITR is: " + pitr) - time.sleep(30) + time.sleep(60) cluster.make_restore(backup,check_pbm_status=True) assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 From d077d2746eeddbd35e0e1d115ad7573214c14572 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Tue, 16 Jul 2024 14:47:01 +0300 Subject: [PATCH 3/7] PBM-1043 print pbm status before the restore --- pbm-functional/pytest/test_PBM-1043.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py index 3e69e52d..163ae55e 100644 --- a/pbm-functional/pytest/test_PBM-1043.py +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -88,5 +88,7 @@ def insert_docs(): backup="--time=" + pitr Cluster.log("Time for PITR is: " + pitr) time.sleep(60) + status=cluster.exec_pbm_cli("status") + Cluster.log(status.stdout) cluster.make_restore(backup,check_pbm_status=True) assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 From b0a2fd7e1f2e52566b833c52e98d3ca0ba7415fb Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Tue, 16 Jul 2024 16:24:16 +0300 Subject: [PATCH 4/7] PBM-1043 more verbose logging --- pbm-functional/pytest/test_PBM-1043.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py index 163ae55e..ca09cb1f 100644 --- a/pbm-functional/pytest/test_PBM-1043.py +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -58,7 +58,10 @@ def insert_docs(): cluster.exec_pbm_cli("config --file=/etc/pbm-1043.conf") time.sleep(30) Cluster.log("Check if PITR is running") - assert cluster.check_pitr() + if not cluster.check_pitr(): + logs=cluster.exec_pbm_cli("logs -sD -t0") + assert False, logs.stdout + Cluster.log("Start inserting docs in the background") background_insert = threading.Thread(target=insert_docs) @@ -73,7 +76,10 @@ def insert_docs(): nrs203.check_output('supervisorctl stop pbm-agent') time.sleep(60) - assert cluster.check_pitr() + Cluster.log("Check if PITR is running") + if not cluster.check_pitr(): + logs=cluster.exec_pbm_cli("logs -sD -t0") + assert False, logs.stdout Cluster.log("Start pbm-agent and mongod") nrs103.check_output('supervisorctl start mongod') @@ -81,7 +87,10 @@ def insert_docs(): time.sleep(60) background_insert.join() - assert cluster.check_pitr() + Cluster.log("Check if PITR is running") + if not cluster.check_pitr(): + logs=cluster.exec_pbm_cli("logs -sD -t0") + assert False, logs.stdout assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") From 0c48bdf9852692ea93e97925d44586ba3c2af7c2 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Tue, 16 Jul 2024 17:45:32 +0300 Subject: [PATCH 5/7] PBM-1043 increase timeout --- pbm-functional/pytest/test_PBM-1043.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py index ca09cb1f..eff83052 100644 --- a/pbm-functional/pytest/test_PBM-1043.py +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -56,7 +56,7 @@ def insert_docs(): cluster.make_backup("logical") cluster.exec_pbm_cli("config --file=/etc/pbm-1043.conf") - time.sleep(30) + time.sleep(60) Cluster.log("Check if PITR is running") if not cluster.check_pitr(): logs=cluster.exec_pbm_cli("logs -sD -t0") From bf357bd03afb2fabd4f8a305b2d49639f0778a86 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Wed, 17 Jul 2024 14:57:33 +0300 Subject: [PATCH 6/7] PBM-1043 remove pitr status checks --- pbm-functional/pytest/test_PBM-1043.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py index eff83052..0d679bc1 100644 --- a/pbm-functional/pytest/test_PBM-1043.py +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -76,10 +76,6 @@ def insert_docs(): nrs203.check_output('supervisorctl stop pbm-agent') time.sleep(60) - Cluster.log("Check if PITR is running") - if not cluster.check_pitr(): - logs=cluster.exec_pbm_cli("logs -sD -t0") - assert False, logs.stdout Cluster.log("Start pbm-agent and mongod") nrs103.check_output('supervisorctl start mongod') @@ -87,17 +83,16 @@ def insert_docs(): time.sleep(60) background_insert.join() - Cluster.log("Check if PITR is running") - if not cluster.check_pitr(): - logs=cluster.exec_pbm_cli("logs -sD -t0") - assert False, logs.stdout assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") backup="--time=" + pitr Cluster.log("Time for PITR is: " + pitr) time.sleep(60) + cluster.disable_pitr() + pymongo.MongoClient(cluster.connection).drop_database('test') status=cluster.exec_pbm_cli("status") Cluster.log(status.stdout) cluster.make_restore(backup,check_pbm_status=True) assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 + assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False) From 8ce3bd8f7b3cc176726a5a72d2312f01a2a1ec46 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Thu, 18 Jul 2024 10:27:40 +0300 Subject: [PATCH 7/7] PBM-1043 Check if oplog slicer has been started on the nodes with the highest prioritie --- pbm-functional/pytest/test_PBM-1043.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pbm-functional/pytest/test_PBM-1043.py b/pbm-functional/pytest/test_PBM-1043.py index 0d679bc1..370b6561 100644 --- a/pbm-functional/pytest/test_PBM-1043.py +++ b/pbm-functional/pytest/test_PBM-1043.py @@ -54,6 +54,7 @@ def insert_docs(): client['test']['test'].insert_one({"doc":i}) time.sleep(1) + cluster.check_pbm_status() cluster.make_backup("logical") cluster.exec_pbm_cli("config --file=/etc/pbm-1043.conf") time.sleep(60) @@ -67,6 +68,11 @@ def insert_docs(): background_insert = threading.Thread(target=insert_docs) background_insert.start() time.sleep(60) + Cluster.log("Check if oplog slicer has been started on the nodes with the highest priorities") + logs=cluster.exec_pbm_cli("logs -sD -t0 -e pitr") + assert '[rscfg/rscfg03:27017] [pitr] created chunk' in logs.stdout + assert '[rs1/rs103:27017] [pitr] created chunk' in logs.stdout + assert '[rs2/rs203:27017] [pitr] created chunk' in logs.stdout nrs103=testinfra.get_host("docker://rs103") nrs203=testinfra.get_host("docker://rs203") @@ -96,3 +102,7 @@ def insert_docs(): cluster.make_restore(backup,check_pbm_status=True) assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 200 assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False) + time.sleep(60) + Cluster.log("PITR must be disabled after the restore") + assert not cluster.check_pitr() + cluster.check_pbm_status()