AssertionError: Error: backup '2024-05-02T05:50:29Z' didn't finish successfully

AssertionError: Starting restore 2024-05-02T05:53:16.904258562Z to point-in-time 2024-05-02T05:51:44 from '2024-05-02T05:50:39Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714629048.1714629102: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714629073,"I":3},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"config.databases","Object":[{"Key":"_id","Value":"test"},{"Key":"primary","Value":"rs1"},{"Key":"partitioned","Value":false},{"Key":"version","Value":[{"Key":"uuid","Value":{"Subtype":4,"Data":"JcoTWAJUT0yNnkazRJ4X8A=="}},{"Key":"lastMod","Value":1}]},{"Key":"lastMovedTimestamp","Value":{"T":1714629073,"I":1}}],"Query":null,"UI":{"Subtype":4,"Data":"JgN3W9cAS3SSFvuJAkYQuw=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (NamespaceNotFound) cannot apply insert or update operation on a non-existent namespace config.databases: { ts: Timestamp(1714629073, 3), t: 1, v: 2, op: "i", ns: "config.databases", o: { _id: "test", primary: "rs1", partitioned: false, version: { uuid: UUID("25ca1358-0254-4f4c-8d9e-46b3449e17f0"), lastMod: 1 }, lastMovedTimestamp: Timestamp(1714629073, 1) }, ui: UUID("2603775b-d700-4b74-9216-fb89024610bb") }

Process completed with exit code 1.

AssertionError: Starting restore 2024-05-02T05:53:24.685828116Z to point-in-time 2024-05-02T05:51:53 from '2024-05-02T05:50:47Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714629056.1714629110: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714629081,"I":2},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"config.databases","Object":[{"Key":"_id","Value":"test"},{"Key":"primary","Value":"rs2"},{"Key":"partitioned","Value":true},{"Key":"version","Value":[{"Key":"uuid","Value":{"Subtype":4,"Data":"GkGmQB6MRdGlKz/PPRielA=="}},{"Key":"timestamp","Value":{"T":1714629080,"I":16}},{"Key":"lastMod","Value":1}]}],"Query":null,"UI":{"Subtype":4,"Data":"dOHuHOzpQSGO139tV4DGmA=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (NamespaceNotFound) cannot apply insert or update operation on a non-existent namespace config.databases: { ts: Timestamp(1714629081, 2), t: 1, v: 2, op: "i", ns: "config.databases", o: { _id: "test", primary: "rs2", partitioned: true, version: { uuid: UUID("1a41a640-1e8c-45d1-a52b-3fcf3d189e94"), timestamp: Timestamp(1714629080, 16), lastMod: 1 } }, ui: UUID("74e1ee1c-ece9-4121-8ed7-7f6d5780c698") }

AssertionError: Starting restore 2024-05-02T06:17:25.030693666Z to point-in-time 2024-05-02T06:16:54 from '2024-05-02T06:16:05Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714630612.1714630642: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714630613,"I":2},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"test.system.views","Object":[{"Key":"_id","Value":"test.test2"},{"Key":"viewOn","Value":"system.buckets.test2"},{"Key":"pipeline","Value":[[{"Key":"$_internalUnpackBucket","Value":[{"Key":"timeField","Value":"timestamp"},{"Key":"metaField","Value":"data"},{"Key":"bucketMaxSpanSeconds","Value":3600},{"Key":"exclude","Value":[]}]}]]}],"Query":null,"UI":{"Subtype":4,"Data":"prK5mVUyRCe7uzdGFLaRTw=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (NamespaceNotFound) Failed to apply operation due to missing collection (a6b2b999-5532-4427-bbbb-374614b6914f): { ts: Timestamp(1714630613, 2), t: 1, v: 2, op: "i", ns: "test.system.views", o: { _id: "test.test2", viewOn: "system.buckets.test2", pipeline: [ { $_internalUnpackBucket: { timeField: "timestamp", metaField: "data", bucketMaxSpanSeconds: 3600, exclude: [] } } ] }, ui: UUID("a6b2b999-5532-4427-bbbb-374614b6914f"), h: 0, wall: new Date(0) }

Process completed with exit code 1.

AssertionError: Starting restore 2024-05-02T05:53:15.54083206Z to point-in-time 2024-05-02T05:51:42 from '2024-05-02T05:50:37Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714629046.1714629100: apply oplog for chunk: applying a transaction entry: apply txn: { "Timestamp": { "T": 1714629071, "I": 2 }, "Term": 1, "Hash": null, "Version": 2, "Operation": "c", "Namespace": "admin.$cmd", "Object": [ { "Key": "applyOps", "Value": [ [ { "Key": "op", "Value": "c" }, { "Key": "ns", "Value": "config.$cmd" }, { "Key": "ui", "Value": { "Subtype": 4, "Data": "TMA+2SEdRBqedzFe8KerOg==" } }, { "Key": "o", "Value": [ { "Key": "create", "Value": "databases" }, { "Key": "idIndex", "Value": [ { "Key": "v", "Value": 2 }, { "Key": "key", "Value": [ { "Key": "_id", "Value": 1 } ] }, { "Key": "name", "Value": "_id_" } ] } ] } ], [ { "Key": "op", "Value": "i" }, { "Key": "ns", "Value": "config.databases" }, { "Key": "ui", "Value": { "Subtype": 4, "Data": "TMA+2SEdRBqedzFe8KerOg==" } }, { "Key": "o", "Value": [ { "Key": "_id", "Value": "test" }, { "Key": "primary", "Value": "rs1" }, { "Key": "partitioned", "Value": false }, { "Key": "version", "Value": [ { "Key": "uuid", "Value": { "Subtype": 4, "Data": "qZW+X2jsSNCq8DJBgTDxPw==" } }, { "Key": "timestamp", "Value": { "T": 1714629070, "I": 16 } }, { "Key": "lastMod", "Value": 1 } ] } ] }, { "Key": "o2", "Value": [ { "Key": "_id", "Value": "test" } ] } ], [ { "Key": "op", "Value": "i" }, { "Key": "ns", "Value": "config.placementHistory" }, { "Key": "ui", "Value": { "Subtype": 4, "Data": "OWhEjU/OS9KhX0KNFjMZNw==" } }, { "Key": "o", "Value": [ { "Key": "_id", "Value": "663329cf8685fa14f7880492" }, { "Key": "nss", "Value": "test" }, { "Key": "timestamp", "Value": { "T": 1714629070, "I": 16 } }, { "Key": "shards", "Value": [ "rs1" ] } ] }, { "Key": "o2", "Value": [ { "Key": "_id", "Value": "663329cf8685fa14f7880492" } ] } ] ] } ], "Query": null, "UI": null, "LSID": "ZgAAAAVpZAAQAAAABOoyQELCDEK1imryxCaP+ZcFdWlkACAAAAAAY5mrDaxi8gv8RmdTsQ+1j7fmkr7JUsabhNmXAheU0fgFdHhuVVVJRAAQAAAABPavEJjOgkCgviDNtS8YWEkA", "TxnNumber": 0, "PrevOpTime": "HAAAABF0cwAAAAAAAAAAABJ0AP//////////AA==" }: applying transaction op: op: {"Timestamp":{"T":1714629071,"I":2},"Term":1,"Hash":null,"Version":0,"Operation":"i","Namespace":"config.databases","Object":[{"Key":"_id","Value":"test"},{"Key":"primary","Value":"rs1"},{"Key":"partitioned","Value":false},{"Key":"version","Value":[{"Key":"uuid","Value":{"Subtype":4,"Data":"qZW+X2jsSNCq8DJBgTDxPw=="}},{"Key":"timestamp","Value":{"T":1714629070,"I":16}},{"Key":"lastMod","Value":1}]}],"Query":[{"Key":"_id","Value":"test"}],"UI":{"Subtype":4,"Data":"TMA+2SEdRBqedzFe8KerOg=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (UnknownError) applyOps had one or more errors applying ops

AssertionError: Starting restore 2024-05-02T06:16:47.458987641Z to point-in-time 2024-05-02T06:16:17 from '2024-05-02T06:15:27Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714630574.1714630604: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714630575,"I":3},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"test.system.views","Object":[{"Key":"_id","Value":"test.test2"},{"Key":"viewOn","Value":"system.buckets.test2"},{"Key":"pipeline","Value":[[{"Key":"$_internalUnpackBucket","Value":[{"Key":"timeField","Value":"timestamp"},{"Key":"metaField","Value":"data"},{"Key":"bucketMaxSpanSeconds","Value":3600}]}]]}],"Query":[{"Key":"_id","Value":"test.test2"}],"UI":{"Subtype":4,"Data":"O9W5zRQLSAaiGQH4J2w0tg=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (UnknownError) applyOps had one or more errors applying ops

AssertionError: Starting restore 2024-05-02T06:26:24.750565616Z from '2024-05-02T06:25:55Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714631155.1714631164: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714631155,"I":4},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"test.system.buckets.test2","Object":[{"Key":"_id","Value":"663331bc0c7325294b817c42"},{"Key":"control","Value":[{"Key":"version","Value":1},{"Key":"min","Value":[{"Key":"_id","Value":"663331f3728c9b8901a88b5c"},{"Key":"timestamp","Value":"2024-05-02T06:25:00Z"}]},{"Key":"max","Value":[{"Key":"_id","Value":"663331f3728c9b8901a88b5c"},{"Key":"timestamp","Value":"2024-05-02T06:25:55.177Z"}]}]},{"Key":"meta","Value":8},{"Key":"data","Value":[{"Key":"_id","Value":[{"Key":"0","Value":"663331f3728c9b8901a88b5c"}]},{"Key":"timestamp","Value":[{"Key":"0","Value":"2024-05-02T06:25:55.177Z"}]}]}],"Query":[{"Key":"meta","Value":8},{"Key":"_id","Value":"663331bc0c7325294b817c42"}],"UI":null,"LSID":"SAAAAAVpZAAQAAAABOuFE0smM0BTrx4go74iLPUFdWlkACAAAAAAY5mrDaxi8gv8RmdTsQ+1j7fmkr7JUsabhNmXAheU0fgA","TxnNumber":1,"PrevOpTime":"HAAAABF0cwAAAAAAAAAAABJ0AP//////////AA=="} | merr <nil>: applyOps: (NamespaceNotFound) cannot apply insert or update operation on a non-existent namespace test.system.buckets.test2: { ts: Timestamp(1714631155, 4), t: 1, v: 2, op: "i", ns: "test.system.buckets.test2", o: { _id: ObjectId('663331bc0c7325294b817c42'), control: { version: 1, min: { _id: ObjectId('663331f3728c9b8901a88b5c'), timestamp: new Date(1714631100000) }, max: { _id: ObjectId('663331f3728c9b8901a88b5c'), timestamp: new Date(1714631155177) } }, meta: 8, data: { _id: { 0: ObjectId('663331f3728c9b8901a88b5c') }, timestamp: { 0: new Date(1714631155177) } } }, o2: { meta: 8, _id: ObjectId('663331bc0c7325294b817c42') }, lsid: { id: UUID("eb85134b-2633-4053-af1e-20a3be222cf5"), uid: BinData(0, 6399AB0DAC62F20BFC466753B10FB58FB7E692BEC952C69B84D997021794D1F8) }, txnNumber: 1, prevOpTime: { ts: Timestamp(0, 0), t: -1 } }

pymongo.errors.OperationFailure: got stale shardVersion response from shard rs1 at host rs101:27017 :: caused by :: timestamp mismatch detected for test.test2, full error: {'ok': 0.0, 'errmsg': 'got stale shardVersion response from shard rs1 at host rs101:27017 :: caused by :: timestamp mismatch detected for test.test2', 'code': 13388, 'codeName': 'StaleConfig', 'ns': 'test.test2', 'vReceived': {'e': ObjectId('6633323369ad5ea9557df733'), 't': Timestamp(1714631219, 59), 'v': Timestamp(1, 3)}, 'vWanted': {'e': ObjectId('000000000000000000000000'), 't': Timestamp(0, 0), 'v': Timestamp(0, 0)}, 'shardId': 'rs1', '$clusterTime': {'clusterTime': Timestamp(1714631263, 21), 'signature': {'hash': b'\xcc\x18V\x135\xbag\x8e\xa1\xfc\xc9\x8e\xfa\xa5u\x93\xafv\x81\x0e', 'keyId': 7364284924406267927}}, 'operationTime': Timestamp(1714631263, 21)}

Process completed with exit code 1.

Failed: Timeout >300.0s

AssertionError: Starting restore 2024-05-02T05:58:28.993931741Z to point-in-time 2024-05-02T05:56:53 from '2024-05-02T05:55:48Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714629357.1714629411: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714629382,"I":2},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"config.databases","Object":[{"Key":"_id","Value":"test"},{"Key":"primary","Value":"rs2"},{"Key":"partitioned","Value":false},{"Key":"version","Value":[{"Key":"uuid","Value":{"Subtype":4,"Data":"uHiElClATNKl2VzF5FMdrQ=="}},{"Key":"timestamp","Value":{"T":1714629381,"I":16}},{"Key":"lastMod","Value":1}]}],"Query":[{"Key":"_id","Value":"test"}],"UI":{"Subtype":4,"Data":"xl3fKjQST/yApT7DyihF9A=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (NamespaceNotFound) cannot apply insert or update operation on a non-existent namespace config.databases: { ts: Timestamp(1714629382, 2), t: 1, v: 2, op: "i", ns: "config.databases", o: { _id: "test", primary: "rs2", partitioned: false, version: { uuid: UUID("b8788494-2940-4cd2-a5d9-5cc5e4531dad"), timestamp: Timestamp(1714629381, 16), lastMod: 1 } }, o2: { _id: "test" }, ui: UUID("c65ddf2a-3412-4ffc-80a5-3ec3ca2845f4") }

AssertionError: Starting restore 2024-05-02T06:23:46.0206853Z to point-in-time 2024-05-02T06:23:15 from '2024-05-02T06:22:26Z'...Started logical restore. Waiting to finish...Error: operation failed with: reply oplog: replay chunk 1714630993.1714631023: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714630994,"I":2},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"test.system.views","Object":[{"Key":"_id","Value":"test.test2"},{"Key":"viewOn","Value":"system.buckets.test2"},{"Key":"pipeline","Value":[[{"Key":"$_internalUnpackBucket","Value":[{"Key":"timeField","Value":"timestamp"},{"Key":"metaField","Value":"data"},{"Key":"bucketMaxSpanSeconds","Value":3600}]}]]}],"Query":[{"Key":"_id","Value":"test.test2"}],"UI":{"Subtype":4,"Data":"GNHN4NqgSKCSWppMdsbLhg=="},"LSID":null,"TxnNumber":null,"PrevOpTime":null} | merr <nil>: applyOps: (NamespaceNotFound) Failed to apply operation due to missing collection (18d1cde0-daa0-48a0-925a-9a4c76c6cb86): { ts: Timestamp(1714630994, 2), t: 1, v: 2, op: "i", ns: "test.system.views", o: { _id: "test.test2", viewOn: "system.buckets.test2", pipeline: [ { $_internalUnpackBucket: { timeField: "timestamp", metaField: "data", bucketMaxSpanSeconds: 3600 } } ] }, o2: { _id: "test.test2" }, ui: UUID("18d1cde0-daa0-48a0-925a-9a4c76c6cb86"), h: 0, wall: new Date(0) }

AssertionError: Starting restore 2024-05-02T06:33:57.973870504Z from '2024-05-02T06:33:32Z'...Started logical restore. Waiting to finish...Error: operation failed with: waiting for dumpDone: cluster failed: reply oplog: replay chunk 1714631612.1714631621: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1714631612,"I":1},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"test.system.buckets.test2","Object":[{"Key":"_id","Value":"6633339cc2db84b88529c482"},{"Key":"control","Value":[{"Key":"version","Value":1},{"Key":"min","Value":[{"Key":"_id","Value":"663333bc5111daadb0bb139a"},{"Key":"timestamp","Value":"2024-05-02T06:33:00Z"}]},{"Key":"max","Value":[{"Key":"_id","Value":"663333bc5111daadb0bb139a"},{"Key":"timestamp","Value":"2024-05-02T06:33:32.002Z"}]}]},{"Key":"meta","Value":9},{"Key":"data","Value":[{"Key":"_id","Value":[{"Key":"0","Value":"663333bc5111daadb0bb139a"}]},{"Key":"timestamp","Value":[{"Key":"0","Value":"2024-05-02T06:33:32.002Z"}]}]}],"Query":[{"Key":"meta","Value":9},{"Key":"_id","Value":"6633339cc2db84b88529c482"}],"UI":null,"LSID":"SAAAAAVpZAAQAAAABKyIOd5o/Uo6kksaoMsoDl8FdWlkACAAAAAAY5mrDaxi8gv8RmdTsQ+1j7fmkr7JUsabhNmXAheU0fgA","TxnNumber":1,"PrevOpTime":"HAAAABF0cwAAAAAAAAAAABJ0AP//////////AA=="} | merr <nil>: applyOps: (NamespaceNotFound) cannot apply insert or update operation on a non-existent namespace test.system.buckets.test2: { ts: Timestamp(1714631612, 1), t: 1, v: 2, op: "i", ns: "test.system.buckets.test2", o: { _id: ObjectId('6633339cc2db84b88529c482'), control: { version: 1, min: { _id: ObjectId('663333bc5111daadb0bb139a'), timestamp: new Date(1714631580000) }, max: { _id: ObjectId('663333bc5111daadb0bb139a'), timestamp: new Date(1714631612002) } }, meta: 9, data: { _id: { 0: ObjectId('663333bc5111daadb0bb139a') }, timestamp: { 0: new Date(1714631612002) } } }, o2: { meta: 9, _id: ObjectId('6633339cc2db84b88529c482') }, lsid: { id: UUID("ac8839de-68fd-4a3a-924b-1aa0cb280e5f"), uid: BinData(0, 6399AB0DAC62F20BFC466753B10FB58FB7E692BEC952C69B84D997021794D1F8) }, txnNumber: 1, prevOpTime: { ts: Timestamp(0, 0), t: -1 } }

pymongo.errors.OperationFailure: got stale shardVersion response from shard rs1 at host rs101:27017 :: caused by :: timestamp mismatch detected for test.test2, full error: {'ok': 0.0, 'errmsg': 'got stale shardVersion response from shard rs1 at host rs101:27017 :: caused by :: timestamp mismatch detected for test.test2', 'code': 13388, 'codeName': 'StaleConfig', 'ns': 'test.test2', 'vReceived': {'t': Timestamp(1714631677, 52), 'e': ObjectId('663333fdd2be4eb253532c69'), 'v': Timestamp(1, 1)}, 'vWanted': {'t': Timestamp(0, 0), 'e': ObjectId('000000000000000000000000'), 'v': Timestamp(0, 0)}, 'shardId': 'rs1', '$clusterTime': {'clusterTime': Timestamp(1714631729, 22), 'signature': {'hash': b'\xca\xba\xaee3\xb3\x1fM~\xccW\x08\x00N\xc8(\x0e\x18\xf9p', 'keyId': 7364286870026453018}}, 'operationTime': Timestamp(1714631729, 22)}