PBM-1265 STR

.github/workflows/PBM-CUSTOM.yml

@@ -18,15 +18,15 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
       PSMDB: ${{ github.event.inputs.psmdb || 'percona/percona-server-mongodb' }}
       PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }}
       GO_VER: ${{ github.event.inputs.go_ver || 'bullseye' }}
       TEST: ${{ github.event.inputs.test || '-k logical --verbose test_sharded.py' }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Setup environment
       run: |
         docker-compose build 
@@ -37,7 +37,7 @@ jobs:
         docker-compose run --rm test pytest -s --junitxml=junit.xml ${{ env.TEST }}
       working-directory: ./pbm-functional/pytest
     - name: Publish Test Report
-      uses: mikepenz/action-junit-report@v3
+      uses: mikepenz/action-junit-report@v4
       if: success() || failure() 
       with:
         report_paths: '**/junit.xml'

.github/workflows/PBM-FULL.yml

@@ -10,10 +10,6 @@ on:
         description: "golang version"
         required: false
 
-#  push:
-#    branches:
-#      - main
-
   pull_request:
     branches:
       - main
@@ -22,7 +18,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 120
     strategy:
       fail-fast: false
@@ -33,7 +29,7 @@ jobs:
       PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }}
       GO_VER: ${{ github.event.inputs.go_ver || 'bullseye' }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Setup environment with PSMDB ${{ matrix.psmdb }} and PBM branch ${{ env.PBM_BRANCH }}
       run: |
         PSMDB=percona/percona-server-mongodb:${{ matrix.psmdb }} docker-compose build
@@ -44,7 +40,7 @@ jobs:
         docker-compose run --rm test pytest -s --junitxml=junit.xml -k ${{ matrix.test }}
       working-directory: ./pbm-functional/pytest
     - name: Publish Test Report
-      uses: mikepenz/action-junit-report@v3
+      uses: mikepenz/action-junit-report@v4
       if: success() || failure() 
       with:
         report_paths: '**/junit.xml'

.github/workflows/PBM.yml

@@ -13,28 +13,20 @@ on:
         description: "golang version"
         required: false
 
-#  push:
-#    branches:
-#      - main
-
-#  pull_request:
-#    branches:
-#      - main
-
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         test: [logical, physical, incremental]
     env:
-      PSMDB: ${{ github.event.inputs.psmdb || 'percona/percona-server-mongodb:4.4' }}
+      PSMDB: ${{ github.event.inputs.psmdb || 'percona/percona-server-mongodb' }}
       PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }}
       GO_VER: ${{ github.event.inputs.go_ver || 'latest' }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Setup environment with PSMDB ${{ env.PSMDB }} and PBM branch ${{ env.PBM_BRANCH }}
       run: |
         docker-compose build
@@ -45,7 +37,7 @@ jobs:
         docker-compose run --rm test pytest -s --junitxml=junit.xml -k ${{ matrix.test }}
       working-directory: ./pbm-functional/pytest
     - name: Publish Test Report
-      uses: mikepenz/action-junit-report@v3
+      uses: mikepenz/action-junit-report@v4
       if: success() || failure() 
       with:
         report_paths: '**/junit.xml'

pbm-functional/pytest/cluster.py

@@ -431,8 +431,12 @@ def make_restore(self, name, **kwargs):
         Cluster.log("Restore started")
         timeout=kwargs.get('timeout', 240)
         result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
-        if result.rc == 124:
+
+        if result.rc == 0:
+            Cluster.log(result.stdout)
+        else:
             # try to catch possible failures if timeout exceeded
+            error=''
             for host in self.mongod_hosts:
                 try:
                     container = docker.from_env().containers.get(host)
@@ -441,14 +445,16 @@ def make_restore(self, name, **kwargs):
                     if get_logs.exit_code == 0:
                         Cluster.log(
                             "!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
-                        Cluster.log(get_logs.output.decode('utf-8'))
+                        logs = get_logs.output.decode('utf-8')
+                        Cluster.log(logs)
+                        if '"s":"F"' in logs:
+                            error = logs
                 except docker.errors.APIError:
                     pass
-            assert False, "Timeout for restore exceeded"
-        elif result.rc == 0:
-            Cluster.log(result.stdout)
-        else:
-            assert False, result.stdout + result.stderr
+            if error:
+                assert False, result.stdout + result.stderr + "\n" + error
+            else:
+                assert False, result.stdout + result.stderr
 
         restart_cluster=kwargs.get('restart_cluster', False)
         if restart_cluster:
@@ -478,7 +484,7 @@ def destroy(self,**kwargs):
         if cleanup:
             result=self.exec_pbm_cli("delete-pitr --all --force --yes ")
             Cluster.log(result.stdout + result.stderr)
-            result=self.exec_pbm_cli("delete-backup --older-than=9999-01-01 --force --yes")
+            result=self.exec_pbm_cli("delete-backup --older-than=0d --force --yes")
             Cluster.log(result.stdout + result.stderr)
         for host in self.all_hosts:
             try:

pbm-functional/pytest/test_PBM-1265.py

@@ -0,0 +1,91 @@
+import pytest
+import pymongo
+import bson
+import testinfra
+import time
+import os
+import docker
+import threading
+import json
+
+from datetime import datetime
+from cluster import Cluster
+
+@pytest.fixture(scope="package")
+def docker_client():
+    return docker.from_env()
+
+@pytest.fixture(scope="package")
+def config():
+    return { "mongos": "mongos",
+             "configserver":
+                            {"_id": "rscfg", "members": [{"host":"rscfg01"},{"host":"rscfg02"},{"host":"rscfg03"}]},
+             "shards":[
+                            {"_id": "rs1", "members": [{"host":"rs101"},{"host":"rs102"},{"host":"rs103"}]},
+                            {"_id": "rs2", "members": [{"host":"rs201"},{"host":"rs202"},{"host":"rs203"}]}
+                      ]}
+
+
+@pytest.fixture(scope="package")
+def cluster(config):
+    return Cluster(config, mongod_extra_args="--setParameter logicalSessionRefreshMillis=180000") # 3minutes
+
+@pytest.fixture(scope="function")
+def start_cluster(cluster,request):
+    try:
+        cluster.destroy()
+        os.chmod("/backups",0o777)
+        os.system("rm -rf /backups/*")
+        cluster.create()
+        cluster.setup_pbm()
+        result = cluster.exec_pbm_cli("config --set storage.type=filesystem --set storage.filesystem.path=/backups --set backup.compression=none --out json")
+        assert result.rc == 0
+        Cluster.log("Setup PBM with fs storage:\n" + result.stdout)
+        client=pymongo.MongoClient(cluster.connection)
+        client.admin.command("enableSharding", "test")
+        client.admin.command("shardCollection", "test.test", key={"_id": "hashed"})
+        yield True
+
+    finally:
+        if request.config.getoption("--verbose"):
+            cluster.get_logs()
+        cluster.destroy()
+        os.system("rm -rf /backups/*")
+
+@pytest.mark.timeout(1200, func_only=True)
+def test_physical_pitr_PBM_T251(start_cluster,cluster):
+    cluster.check_pbm_status()
+    for i in range(30):
+         pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i})
+    cluster.make_backup("logical")
+    cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=3")
+    backup=cluster.make_backup("physical")
+    for i in range(30):
+         pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"b": i})
+    timeout = time.time() + 360
+    while True:
+        status = cluster.get_status()
+        for snapshot in status['backups']['snapshot']:
+            if snapshot['name'] == backup:
+                backup_epoch = snapshot['restoreTo']
+        if 'pitrChunks' in status['backups']:
+            if 'pitrChunks' in status['backups']['pitrChunks']:
+                pitr_epoch = status['backups']['pitrChunks']['pitrChunks'][-1]['range']['end']
+                if pitr_epoch > backup_epoch:
+                    pitr = datetime.utcfromtimestamp(pitr_epoch).strftime("%Y-%m-%dT%H:%M:%S")
+                    Cluster.log("Time for PITR is: " + pitr)
+                    break
+        if time.time() > timeout:
+            assert False
+        time.sleep(1)
+    time.sleep(30)
+    backup="--time=" + pitr + " --base-snapshot=" + backup
+    cluster.disable_pitr()
+    cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True)
+    assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 60
+
+
+    backup=cluster.make_backup("physical")
+    cluster.make_restore(backup,restart_cluster=True,check_pbm_status=True)
+    assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 60
+