From b01c5fbef9ed4b7fe56451cf4da89bee6448a87a Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Thu, 28 Mar 2024 17:29:52 +0200 Subject: [PATCH 1/5] PBM-1265 STR --- pbm-functional/pytest/cluster.py | 11 ++-- pbm-functional/pytest/test_PBM-1265.py | 89 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 pbm-functional/pytest/test_PBM-1265.py diff --git a/pbm-functional/pytest/cluster.py b/pbm-functional/pytest/cluster.py index 01bbc80e..0ec21f09 100644 --- a/pbm-functional/pytest/cluster.py +++ b/pbm-functional/pytest/cluster.py @@ -431,7 +431,10 @@ def make_restore(self, name, **kwargs): Cluster.log("Restore started") timeout=kwargs.get('timeout', 240) result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait') - if result.rc == 124: + + if result.rc == 0: + Cluster.log(result.stdout) + else: # try to catch possible failures if timeout exceeded for host in self.mongod_hosts: try: @@ -444,10 +447,6 @@ def make_restore(self, name, **kwargs): Cluster.log(get_logs.output.decode('utf-8')) except docker.errors.APIError: pass - assert False, "Timeout for restore exceeded" - elif result.rc == 0: - Cluster.log(result.stdout) - else: assert False, result.stdout + result.stderr restart_cluster=kwargs.get('restart_cluster', False) @@ -478,7 +477,7 @@ def destroy(self,**kwargs): if cleanup: result=self.exec_pbm_cli("delete-pitr --all --force --yes ") Cluster.log(result.stdout + result.stderr) - result=self.exec_pbm_cli("delete-backup --older-than=9999-01-01 --force --yes") + result=self.exec_pbm_cli("delete-backup --older-than=0 --force --yes") Cluster.log(result.stdout + result.stderr) for host in self.all_hosts: try: diff --git a/pbm-functional/pytest/test_PBM-1265.py b/pbm-functional/pytest/test_PBM-1265.py new file mode 100644 index 00000000..a100e63a --- /dev/null +++ b/pbm-functional/pytest/test_PBM-1265.py @@ -0,0 +1,89 @@ +import pytest +import pymongo +import bson +import testinfra +import time +import os +import docker +import threading +import json + +from datetime import datetime +from cluster import Cluster + +@pytest.fixture(scope="package") +def docker_client(): + return docker.from_env() + +@pytest.fixture(scope="package") +def config(): + return { "mongos": "mongos", + "configserver": + {"_id": "rscfg", "members": [{"host":"rscfg01"},{"host": "rscfg02"},{"host": "rscfg03" }]}, + "shards":[ + {"_id": "rs1", "members": [{"host":"rs101"},{"host": "rs102"},{"host": "rs103" }]}, + {"_id": "rs2", "members": [{"host":"rs201"},{"host": "rs202"},{"host": "rs203" }]}, + {"_id": "rs3", "members": [{"host":"rs301"},{"host": "rs302"},{"host": "rs303" }]} + ]} + +@pytest.fixture(scope="package") +def cluster(config): + return Cluster(config) + +@pytest.fixture(scope="function") +def start_cluster(cluster,request): + try: + cluster.destroy() + os.chmod("/backups",0o777) + os.system("rm -rf /backups/*") + cluster.create() + cluster.setup_pbm() + client=pymongo.MongoClient(cluster.connection) + client.admin.command("enableSharding", "test") + client.admin.command("shardCollection", "test.test", key={"_id": "hashed"}) + yield True + + finally: + if request.config.getoption("--verbose"): + cluster.get_logs() + cluster.destroy(cleanup_backups=True) + +#@pytest.mark.timeout(600, func_only=True) +def test_physical_pitr(start_cluster,cluster): + cluster.check_pbm_status() + cluster.exec_pbm_cli("config --set pitr.enabled=true --set pitr.compression=none --set pitr.oplogSpanMin=5") + for i in range(30): + pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i}) + cluster.make_backup("logical") + backup=cluster.make_backup("physical") + for i in range(30): + pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"b": i}) + timeout = time.time() + 360 + while True: + status = cluster.get_status() + for snapshot in status['backups']['snapshot']: + if snapshot['name'] == backup: + backup_epoch = snapshot['restoreTo'] + if 'pitrChunks' in status['backups']: + if 'pitrChunks' in status['backups']['pitrChunks']: + pitr_epoch = status['backups']['pitrChunks']['pitrChunks'][-1]['range']['end'] + if pitr_epoch > backup_epoch: + pitr = datetime.utcfromtimestamp(pitr_epoch).strftime("%Y-%m-%dT%H:%M:%S") + Cluster.log("Time for PITR is: " + pitr) + break + if time.time() > timeout: + assert False + time.sleep(1) + time.sleep(60) + backup="--time=" + pitr + " --base-snapshot=" + backup + cluster.disable_pitr() + cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True) + assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 60 + + cluster.exec_pbm_cli("config --set pitr.enabled=true --set pitr.compression=none --set pitr.oplogSpanMin=5") + cluster.make_backup("logical") + backup=cluster.make_backup("physical") + cluster.disable_pitr() + cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True) + assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 60 + From f63986ddfa3b84c536292d071ef1c6e59f5c9756 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Thu, 28 Mar 2024 22:30:15 +0200 Subject: [PATCH 2/5] PBM-1265 simplify, fix cleanup --- pbm-functional/pytest/cluster.py | 2 +- pbm-functional/pytest/test_PBM-1265.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pbm-functional/pytest/cluster.py b/pbm-functional/pytest/cluster.py index 0ec21f09..0ba7d210 100644 --- a/pbm-functional/pytest/cluster.py +++ b/pbm-functional/pytest/cluster.py @@ -477,7 +477,7 @@ def destroy(self,**kwargs): if cleanup: result=self.exec_pbm_cli("delete-pitr --all --force --yes ") Cluster.log(result.stdout + result.stderr) - result=self.exec_pbm_cli("delete-backup --older-than=0 --force --yes") + result=self.exec_pbm_cli("delete-backup --older-than=0d --force --yes") Cluster.log(result.stdout + result.stderr) for host in self.all_hosts: try: diff --git a/pbm-functional/pytest/test_PBM-1265.py b/pbm-functional/pytest/test_PBM-1265.py index a100e63a..c16191f3 100644 --- a/pbm-functional/pytest/test_PBM-1265.py +++ b/pbm-functional/pytest/test_PBM-1265.py @@ -19,16 +19,16 @@ def docker_client(): def config(): return { "mongos": "mongos", "configserver": - {"_id": "rscfg", "members": [{"host":"rscfg01"},{"host": "rscfg02"},{"host": "rscfg03" }]}, + {"_id": "rscfg", "members": [{"host":"rscfg01"},{"host":"rscfg02"},{"host":"rscfg03"}]}, "shards":[ - {"_id": "rs1", "members": [{"host":"rs101"},{"host": "rs102"},{"host": "rs103" }]}, - {"_id": "rs2", "members": [{"host":"rs201"},{"host": "rs202"},{"host": "rs203" }]}, - {"_id": "rs3", "members": [{"host":"rs301"},{"host": "rs302"},{"host": "rs303" }]} + {"_id": "rs1", "members": [{"host":"rs101"},{"host":"rs102"},{"host":"rs103"}]}, + {"_id": "rs2", "members": [{"host":"rs201"},{"host":"rs202"},{"host":"rs203"}]} ]} + @pytest.fixture(scope="package") def cluster(config): - return Cluster(config) + return Cluster(config, mongod_extra_args="--setParameter logicalSessionRefreshMillis=120000") # 2minutes @pytest.fixture(scope="function") def start_cluster(cluster,request): @@ -38,6 +38,9 @@ def start_cluster(cluster,request): os.system("rm -rf /backups/*") cluster.create() cluster.setup_pbm() + result = cluster.exec_pbm_cli("config --set storage.type=filesystem --set storage.filesystem.path=/backups --set backup.compression=none --out json") + assert result.rc == 0 + Cluster.log("Setup PBM with fs storage:\n" + result.stdout) client=pymongo.MongoClient(cluster.connection) client.admin.command("enableSharding", "test") client.admin.command("shardCollection", "test.test", key={"_id": "hashed"}) @@ -46,15 +49,16 @@ def start_cluster(cluster,request): finally: if request.config.getoption("--verbose"): cluster.get_logs() - cluster.destroy(cleanup_backups=True) + cluster.destroy() + os.system("rm -rf /backups/*") -#@pytest.mark.timeout(600, func_only=True) +@pytest.mark.timeout(1200, func_only=True) def test_physical_pitr(start_cluster,cluster): cluster.check_pbm_status() - cluster.exec_pbm_cli("config --set pitr.enabled=true --set pitr.compression=none --set pitr.oplogSpanMin=5") for i in range(30): pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i}) cluster.make_backup("logical") + cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=2") backup=cluster.make_backup("physical") for i in range(30): pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"b": i}) @@ -74,16 +78,13 @@ def test_physical_pitr(start_cluster,cluster): if time.time() > timeout: assert False time.sleep(1) - time.sleep(60) backup="--time=" + pitr + " --base-snapshot=" + backup cluster.disable_pitr() cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True) assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 60 - cluster.exec_pbm_cli("config --set pitr.enabled=true --set pitr.compression=none --set pitr.oplogSpanMin=5") - cluster.make_backup("logical") + backup=cluster.make_backup("physical") - cluster.disable_pitr() cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True) assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 60 From 13449c1c1b5a0da9bd4d50446eb05a0afd4768fc Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Thu, 28 Mar 2024 23:53:20 +0200 Subject: [PATCH 3/5] PBM-1265 increased timeouts for GH --- pbm-functional/pytest/test_PBM-1265.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pbm-functional/pytest/test_PBM-1265.py b/pbm-functional/pytest/test_PBM-1265.py index c16191f3..48daff1f 100644 --- a/pbm-functional/pytest/test_PBM-1265.py +++ b/pbm-functional/pytest/test_PBM-1265.py @@ -28,7 +28,7 @@ def config(): @pytest.fixture(scope="package") def cluster(config): - return Cluster(config, mongod_extra_args="--setParameter logicalSessionRefreshMillis=120000") # 2minutes + return Cluster(config, mongod_extra_args="--setParameter logicalSessionRefreshMillis=180000") # 3minutes @pytest.fixture(scope="function") def start_cluster(cluster,request): @@ -58,7 +58,7 @@ def test_physical_pitr(start_cluster,cluster): for i in range(30): pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i}) cluster.make_backup("logical") - cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=2") + cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=3") backup=cluster.make_backup("physical") for i in range(30): pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"b": i}) @@ -78,6 +78,7 @@ def test_physical_pitr(start_cluster,cluster): if time.time() > timeout: assert False time.sleep(1) + time.sleep(30) backup="--time=" + pitr + " --base-snapshot=" + backup cluster.disable_pitr() cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True) From c1f2f7695c9aa72d11da1b74af9b7a79003239b5 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Fri, 29 Mar 2024 09:47:30 +0200 Subject: [PATCH 4/5] PBM-1265 zephyr test case --- pbm-functional/pytest/cluster.py | 11 +++++++++-- pbm-functional/pytest/test_PBM-1265.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pbm-functional/pytest/cluster.py b/pbm-functional/pytest/cluster.py index 0ba7d210..1a682c75 100644 --- a/pbm-functional/pytest/cluster.py +++ b/pbm-functional/pytest/cluster.py @@ -436,6 +436,7 @@ def make_restore(self, name, **kwargs): Cluster.log(result.stdout) else: # try to catch possible failures if timeout exceeded + error='' for host in self.mongod_hosts: try: container = docker.from_env().containers.get(host) @@ -444,10 +445,16 @@ def make_restore(self, name, **kwargs): if get_logs.exit_code == 0: Cluster.log( "!!!!Possible failure on {}, file pbm.restore.log was found:".format(host)) - Cluster.log(get_logs.output.decode('utf-8')) + logs = get_logs.output.decode('utf-8') + Cluster.log(logs) + if '"s":"F"' in logs: + error = logs except docker.errors.APIError: pass - assert False, result.stdout + result.stderr + if error: + assert False, result.stdout + result.stderr + "\n" + error + else: + assert False, result.stdout + result.stderr restart_cluster=kwargs.get('restart_cluster', False) if restart_cluster: diff --git a/pbm-functional/pytest/test_PBM-1265.py b/pbm-functional/pytest/test_PBM-1265.py index 48daff1f..07432e0e 100644 --- a/pbm-functional/pytest/test_PBM-1265.py +++ b/pbm-functional/pytest/test_PBM-1265.py @@ -53,7 +53,7 @@ def start_cluster(cluster,request): os.system("rm -rf /backups/*") @pytest.mark.timeout(1200, func_only=True) -def test_physical_pitr(start_cluster,cluster): +def test_physical_pitr_PBM_T251(start_cluster,cluster): cluster.check_pbm_status() for i in range(30): pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i}) From efbfb92ad65a94d05725be289b450ddaf8550c12 Mon Sep 17 00:00:00 2001 From: Oleksandr Havryliak Date: Fri, 29 Mar 2024 11:08:49 +0200 Subject: [PATCH 5/5] Update CI to latest --- .github/workflows/PBM-CUSTOM.yml | 6 +++--- .github/workflows/PBM-FULL.yml | 10 +++------- .github/workflows/PBM.yml | 16 ++++------------ 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/.github/workflows/PBM-CUSTOM.yml b/.github/workflows/PBM-CUSTOM.yml index ba67df98..bf889f28 100644 --- a/.github/workflows/PBM-CUSTOM.yml +++ b/.github/workflows/PBM-CUSTOM.yml @@ -18,7 +18,7 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 60 env: PSMDB: ${{ github.event.inputs.psmdb || 'percona/percona-server-mongodb' }} @@ -26,7 +26,7 @@ jobs: GO_VER: ${{ github.event.inputs.go_ver || 'bullseye' }} TEST: ${{ github.event.inputs.test || '-k logical --verbose test_sharded.py' }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup environment run: | docker-compose build @@ -37,7 +37,7 @@ jobs: docker-compose run --rm test pytest -s --junitxml=junit.xml ${{ env.TEST }} working-directory: ./pbm-functional/pytest - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 + uses: mikepenz/action-junit-report@v4 if: success() || failure() with: report_paths: '**/junit.xml' diff --git a/.github/workflows/PBM-FULL.yml b/.github/workflows/PBM-FULL.yml index 80fb4b15..1043f25e 100644 --- a/.github/workflows/PBM-FULL.yml +++ b/.github/workflows/PBM-FULL.yml @@ -10,10 +10,6 @@ on: description: "golang version" required: false -# push: -# branches: -# - main - pull_request: branches: - main @@ -22,7 +18,7 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 120 strategy: fail-fast: false @@ -33,7 +29,7 @@ jobs: PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }} GO_VER: ${{ github.event.inputs.go_ver || 'bullseye' }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup environment with PSMDB ${{ matrix.psmdb }} and PBM branch ${{ env.PBM_BRANCH }} run: | PSMDB=percona/percona-server-mongodb:${{ matrix.psmdb }} docker-compose build @@ -44,7 +40,7 @@ jobs: docker-compose run --rm test pytest -s --junitxml=junit.xml -k ${{ matrix.test }} working-directory: ./pbm-functional/pytest - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 + uses: mikepenz/action-junit-report@v4 if: success() || failure() with: report_paths: '**/junit.xml' diff --git a/.github/workflows/PBM.yml b/.github/workflows/PBM.yml index 89c5ae77..32caae56 100644 --- a/.github/workflows/PBM.yml +++ b/.github/workflows/PBM.yml @@ -13,28 +13,20 @@ on: description: "golang version" required: false -# push: -# branches: -# - main - -# pull_request: -# branches: -# - main - jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false matrix: test: [logical, physical, incremental] env: - PSMDB: ${{ github.event.inputs.psmdb || 'percona/percona-server-mongodb:4.4' }} + PSMDB: ${{ github.event.inputs.psmdb || 'percona/percona-server-mongodb' }} PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }} GO_VER: ${{ github.event.inputs.go_ver || 'latest' }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup environment with PSMDB ${{ env.PSMDB }} and PBM branch ${{ env.PBM_BRANCH }} run: | docker-compose build @@ -45,7 +37,7 @@ jobs: docker-compose run --rm test pytest -s --junitxml=junit.xml -k ${{ matrix.test }} working-directory: ./pbm-functional/pytest - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 + uses: mikepenz/action-junit-report@v4 if: success() || failure() with: report_paths: '**/junit.xml'