From 61886cb9525ac1e27b17142c34fa53f102b915df Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Tue, 10 Jan 2023 12:26:33 -0500 Subject: [PATCH 01/24] wip test --- .../pool-checkout-error-network.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml new file mode 100644 index 0000000000..16d4ac61bf --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml @@ -0,0 +1,39 @@ +version: 1 +style: integration +description: must properly handle network error on check out +runOn: + - + # required for appName in fail point + minServerVersion: "4.9.0" +failPoint: + configureFailPoint: failCommand + # high amount to ensure not interfered with by monitor checks. + mode: { times: 50 } + data: + failCommands: ["isMaster","hello"] + closeConnection: true + appName: &appName "poolCheckOutErrorTest" +poolOptions: + appName: *appName +operations: + - name: ready + - name: checkOut +error: + type: Network +events: + - type: ConnectionCheckOutStarted + address: 42 + - type: ConnectionCreated + address: 42 + - type: ConnectionPoolCleared + address: 42 + - type: ConnectionClosed + address: 42 + reason: error + - type: ConnectionCheckOutFailed + address: 42 + connectionId: 42 + reason: connectionError +ignore: + - ConnectionPoolCreated + - ConnectionPoolReady From f0b33d86b0cf6cc6a92a0a7527376066ca28b1b6 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Wed, 11 Jan 2023 16:47:55 -0500 Subject: [PATCH 02/24] wip tests --- .../tests/cmap-format/README.rst | 21 ++++ .../cmap-format/pool-checkout-error-auth.json | 67 +++++++++++ .../cmap-format/pool-checkout-error-auth.yml | 39 +++++++ .../pool-checkout-error-checkin.json | 101 +++++++++++++++++ .../pool-checkout-error-checkin.yml | 56 +++++++++ .../pool-checkout-error-clears-waitqueue.json | 106 ++++++++++++++++++ .../pool-checkout-error-clears-waitqueue.yml | 65 +++++++++++ .../pool-checkout-error-network.json | 67 +++++++++++ .../pool-create-min-size-error.json | 8 +- .../pool-create-min-size-error.yml | 4 +- 10 files changed, 528 insertions(+), 6 deletions(-) create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst b/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst index 5bb72dd0fe..229a30bf0d 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst @@ -126,11 +126,32 @@ the addition of the following fields to each test: it should be assumed that there is no upper bound on the required server version. + - ``auth`` (optional): If true, the tests MUST only run if authentication + is enabled. If false, tests MUST only run if authentication is not enabled. + If this field is omitted, there is no authentication requirement. + - ``failPoint``: optional, a document containing a ``configureFailPoint`` command to run against the endpoint being used for the test. - ``poolOptions.appName`` (optional): appName attribute to be set in connections, which will be affected by the fail point. +The integration also supports the following additional operations which are not available +in the unit test format: + +failPoint +~~~~~~~~~ + +The ``failPoint`` operation instructs the test runner to configure a fail point +on the endpoint being used for the test. + +The following arguments are supported: + +- ``failPoint``: Required document. The ``configureFailPoint`` command to be + executed. + +When executing this operation, the test runner MUST keep a record of the fail +point so that it can be disabled after the test. + Spec Test Match Function ======================== diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json new file mode 100644 index 0000000000..c966a3f160 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json @@ -0,0 +1,67 @@ +{ + "version": 1, + "style": "integration", + "description": "must properly handle auth error on check out", + "runOn": [ + { + "minServerVersion": "4.9.0", + "auth": true + } + ], + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "saslContinue" + ], + "appName": "poolCheckOutAuthErrorTest", + "errorCode": 18 + } + }, + "poolOptions": { + "appName": "poolCheckOutAuthErrorTest" + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "checkOut" + } + ], + "error": { + "type": "AuthenticationError" + }, + "events": [ + { + "type": "ConnectionCheckOutStarted", + "address": 42 + }, + { + "type": "ConnectionCreated", + "address": 42 + }, + { + "type": "ConnectionPoolCleared", + "address": 42 + }, + { + "type": "ConnectionClosed", + "address": 42, + "reason": "error" + }, + { + "type": "ConnectionCheckOutFailed", + "address": 42, + "connectionId": 42, + "reason": "connectionError" + } + ], + "ignore": [ + "ConnectionPoolCreated", + "ConnectionPoolReady" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml new file mode 100644 index 0000000000..d9cc49ce32 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml @@ -0,0 +1,39 @@ +version: 1 +style: integration +description: must properly handle auth error on check out +runOn: + - + # required for appName in fail point + minServerVersion: "4.9.0" + auth: true +failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["saslContinue"] + appName: &appName "poolCheckOutAuthErrorTest" + errorCode: 18 +poolOptions: + appName: *appName +operations: + - name: ready + - name: checkOut +error: + type: AuthenticationError +events: + - type: ConnectionCheckOutStarted + address: 42 + - type: ConnectionCreated + address: 42 + - type: ConnectionPoolCleared + address: 42 + - type: ConnectionClosed + address: 42 + reason: error + - type: ConnectionCheckOutFailed + address: 42 + connectionId: 42 + reason: connectionError +ignore: + - ConnectionPoolCreated + - ConnectionPoolReady diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json new file mode 100644 index 0000000000..f6dee6f6da --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json @@ -0,0 +1,101 @@ +{ + "version": 1, + "style": "integration", + "description": "must properly handle network error on check out", + "runOn": [ + { + "minServerVersion": "4.9.0" + } + ], + "poolOptions": { + "appName": "poolCheckOutErrorCheckInTest" + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "checkOut", + "label": "conn" + }, + { + "name": "failPoint", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 10 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "poolCheckOutErrorCheckInTest" + } + } + }, + { + "name": "start", + "target": "thread1" + }, + { + "name": "checkOut", + "thread": "thread1" + }, + { + "name": "waitForEvent", + "event": "CheckOutFailed" + }, + { + "name": "checkIn", + "connection": "conn" + } + ], + "events": [ + { + "type": "ConnectionCheckOutStarted", + "address": 42 + }, + { + "type": "ConnectionCreated", + "address": 42 + }, + { + "type": "ConnectionReady", + "address": 42 + }, + { + "type": "ConnectionCheckedOut", + "address": 42 + }, + { + "type": "ConnectionCheckOutStarted" + }, + { + "type": "ConnectionCreated" + }, + { + "type": "ConnectionPoolCleared" + }, + { + "type": "ConnectionClosed", + "reason": "error" + }, + { + "type": "CheckOutFailed", + "reason": "connectionError" + }, + { + "type": "ConnectionCheckedIn" + }, + { + "type": "ConnectionClosed", + "reason": "stale" + } + ], + "ignore": [ + "ConnectionPoolCreated", + "ConnectionPoolReady" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml new file mode 100644 index 0000000000..2d539d29dd --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml @@ -0,0 +1,56 @@ +version: 1 +style: integration +description: must properly handle network error on check out +runOn: + - + # required for appName in fail point + minServerVersion: "4.9.0" +poolOptions: + appName: &appName "poolCheckOutErrorCheckInTest" +operations: + - name: ready + - name: checkOut + label: conn + # Enable a failpoint so the next checkOut will fail. + - name: failPoint + failPoint: + configureFailPoint: failCommand + # Use a high number of times to prevent monitor checks from interfering + mode: { times: 10 } + data: + failCommands: ["isMaster","hello"] + closeConnection: true + appName: *appName + # Perform check out in another thread so the main thread can continue to + # execute operations. + - name: start + target: thread1 + - name: checkOut + thread: thread1 + # Wait for the check out to fail + - name: waitForEvent + event: CheckOutFailed + - name: checkIn + connection: conn +events: + - type: ConnectionCheckOutStarted + address: 42 + - type: ConnectionCreated + address: 42 + - type: ConnectionReady + address: 42 + - type: ConnectionCheckedOut + address: 42 + - type: ConnectionCheckOutStarted + - type: ConnectionCreated + - type: ConnectionPoolCleared + - type: ConnectionClosed + reason: error + - type: CheckOutFailed + reason: connectionError + - type: ConnectionCheckedIn + - type: ConnectionClosed + reason: stale +ignore: + - ConnectionPoolCreated + - ConnectionPoolReady diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json new file mode 100644 index 0000000000..ea7fe1e23f --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json @@ -0,0 +1,106 @@ +{ + "version": 1, + "style": "integration", + "description": "connection error during checkout clears wait queue", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 50 + }, + "data": { + "failCommands": [ + "hello", + "isMaster" + ], + "appName": "poolCheckOutErrorWaitQueueTest", + "closeConnection": true, + "blockConnection": true, + "blockTimeMS": 1000 + } + }, + "poolOptions": { + "maxPoolSize": 1, + "waitQueueTimeoutMS": 30000 + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "start", + "target": "thread1" + }, + { + "name": "checkOut", + "thread": "thread1" + }, + { + "name": "start", + "target": "thread2" + }, + { + "name": "checkOut", + "thread": "thread2" + }, + { + "name": "start", + "target": "thread3" + }, + { + "name": "checkOut", + "thread": "thread3" + }, + { + "name": "waitForEvent", + "event": "ConnectionCheckOutStarted", + "count": 3 + }, + { + "name": "waitForEvent", + "event": "ConnectionCheckOutFailed", + "count": 3, + "timeout": 2000 + } + ], + "events": [ + { + "type": "ConnectionCheckOutStarted", + "address": 42 + }, + { + "type": "ConnectionCheckOutStarted", + "address": 42 + }, + { + "type": "ConnectionCheckOutStarted", + "address": 42 + }, + { + "type": "ConnectionPoolCleared", + "address": 42 + }, + { + "type": "ConnectionCheckOutFailed", + "reason": "connectionError", + "address": 42 + }, + { + "type": "ConnectionCheckOutFailed", + "reason": "connectionError", + "address": 42 + }, + { + "type": "ConnectionCheckOutFailed", + "reason": "connectionError", + "address": 42 + } + ], + "ignore": [ + "ConnectionPoolReady", + "ConnectionCreated", + "ConnectionPoolCreated", + "ConnectionReady", + "ConnectionCheckedIn", + "ConnectionClosed" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml new file mode 100644 index 0000000000..481657e309 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml @@ -0,0 +1,65 @@ +version: 1 +style: integration +description: connection error during checkout clears wait queue +failPoint: + configureFailPoint: failCommand + # high amount to ensure not interfered with by monitor checks. + mode: { times: 50 } + data: + failCommands: ["hello", "isMaster"] + appName: &appName "poolCheckOutErrorWaitQueueTest" + closeConnection: true + blockConnection: true + blockTimeMS: 1000 +poolOptions: + maxPoolSize: 1 + waitQueueTimeoutMS: 30000 +operations: + - name: ready + # start a few threads that all will enter + # the wait queue + - name: start + target: thread1 + - name: checkOut + thread: thread1 + - name: start + target: thread2 + - name: checkOut + thread: thread2 + - name: start + target: thread3 + - name: checkOut + thread: thread3 + # wait for the 3 threads to start checking out a connection + - name: waitForEvent + event: ConnectionCheckOutStarted + count: 3 + - name: waitForEvent + event: ConnectionCheckOutFailed + count: 3 + timeout: 2000 +events: + - type: ConnectionCheckOutStarted + address: 42 + - type: ConnectionCheckOutStarted + address: 42 + - type: ConnectionCheckOutStarted + address: 42 + - type: ConnectionPoolCleared + address: 42 + - type: ConnectionCheckOutFailed + reason: connectionError + address: 42 + - type: ConnectionCheckOutFailed + reason: connectionError + address: 42 + - type: ConnectionCheckOutFailed + reason: connectionError + address: 42 +ignore: + - ConnectionPoolReady + - ConnectionCreated + - ConnectionPoolCreated + - ConnectionReady + - ConnectionCheckedIn + - ConnectionClosed diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json new file mode 100644 index 0000000000..43ecd2dc2f --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json @@ -0,0 +1,67 @@ +{ + "version": 1, + "style": "integration", + "description": "must properly handle network error on check out", + "runOn": [ + { + "minServerVersion": "4.9.0" + } + ], + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 50 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "poolCheckOutErrorTest" + } + }, + "poolOptions": { + "appName": "poolCheckOutErrorTest" + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "checkOut" + } + ], + "error": { + "type": "Network" + }, + "events": [ + { + "type": "ConnectionCheckOutStarted", + "address": 42 + }, + { + "type": "ConnectionCreated", + "address": 42 + }, + { + "type": "ConnectionPoolCleared", + "address": 42 + }, + { + "type": "ConnectionClosed", + "address": 42, + "reason": "error" + }, + { + "type": "ConnectionCheckOutFailed", + "address": 42, + "connectionId": 42, + "reason": "connectionError" + } + ], + "ignore": [ + "ConnectionPoolCreated", + "ConnectionPoolReady" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json index 1c744b850c..509b2a2356 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json @@ -49,15 +49,15 @@ "type": "ConnectionCreated", "address": 42 }, + { + "type": "ConnectionPoolCleared", + "address": 42 + }, { "type": "ConnectionClosed", "address": 42, "connectionId": 42, "reason": "error" - }, - { - "type": "ConnectionPoolCleared", - "address": 42 } ], "ignore": [ diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml index dd5890b1d9..f43c4ee154 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml @@ -30,11 +30,11 @@ events: address: 42 - type: ConnectionCreated address: 42 + - type: ConnectionPoolCleared + address: 42 - type: ConnectionClosed address: 42 connectionId: 42 reason: error - - type: ConnectionPoolCleared - address: 42 ignore: - ConnectionPoolCreated From fc6544ffedae47427824d70e54a3bbb3b63b4f02 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 12 Jan 2023 18:23:14 -0500 Subject: [PATCH 03/24] more work on tests --- .../tests/README.rst | 4 +--- .../cmap-format/pool-checkout-error-auth.json | 3 ++- .../pool-checkout-error-checkin.json | 23 +++++++++---------- .../pool-checkout-error-checkin.yml | 13 +++++------ .../pool-checkout-error-clears-waitqueue.json | 3 ++- .../pool-checkout-error-clears-waitqueue.yml | 1 + .../pool-checkout-error-network.json | 4 ++-- .../pool-checkout-error-network.yml | 4 ++-- 8 files changed, 27 insertions(+), 28 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/README.rst b/source/connection-monitoring-and-pooling/tests/README.rst index ae4af543f2..08d5141494 100644 --- a/source/connection-monitoring-and-pooling/tests/README.rst +++ b/source/connection-monitoring-and-pooling/tests/README.rst @@ -26,11 +26,9 @@ The following tests have not yet been automated, but MUST still be tested: #. All ConnectionPoolOptions MUST be the same for all pools created by a MongoClient #. A user MUST be able to specify all ConnectionPoolOptions via a URI string #. A user MUST be able to subscribe to Connection Monitoring Events in a manner idiomatic to their language and driver -#. When a check out attempt fails because connection set up throws an error, - assert that a ConnectionCheckOutFailedEvent with reason="connectionError" is emitted. Logging Tests ============= Tests for connection pool logging can be found in the `/logging <./logging>`__ subdirectory and are written in the -`Unified Test Format <../../unified-test-format/unified-test-format.rst>`__. \ No newline at end of file +`Unified Test Format <../../unified-test-format/unified-test-format.rst>`__. diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json index c966a3f160..edd22728c1 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json @@ -33,7 +33,8 @@ } ], "error": { - "type": "AuthenticationError" + "type": "AuthenticationError", + "message": "????????" }, "events": [ { diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json index f6dee6f6da..cda624ba59 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json @@ -1,7 +1,7 @@ { "version": 1, "style": "integration", - "description": "must properly handle network error on check out", + "description": "checking in a connection after checkout fails closes connection", "runOn": [ { "minServerVersion": "4.9.0" @@ -45,7 +45,8 @@ }, { "name": "waitForEvent", - "event": "CheckOutFailed" + "event": "ConnectionCheckOutFailed", + "count": 1 }, { "name": "checkIn", @@ -54,20 +55,16 @@ ], "events": [ { - "type": "ConnectionCheckOutStarted", - "address": 42 + "type": "ConnectionCheckOutStarted" }, { - "type": "ConnectionCreated", - "address": 42 + "type": "ConnectionCreated" }, { - "type": "ConnectionReady", - "address": 42 + "type": "ConnectionReady" }, { - "type": "ConnectionCheckedOut", - "address": 42 + "type": "ConnectionCheckedOut" }, { "type": "ConnectionCheckOutStarted" @@ -80,10 +77,11 @@ }, { "type": "ConnectionClosed", - "reason": "error" + "reason": "error", + "connectionId": 2 }, { - "type": "CheckOutFailed", + "type": "ConnectionCheckOutFailed", "reason": "connectionError" }, { @@ -91,6 +89,7 @@ }, { "type": "ConnectionClosed", + "connectionId": 1, "reason": "stale" } ], diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml index 2d539d29dd..a2d2a47e50 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml @@ -1,6 +1,6 @@ version: 1 style: integration -description: must properly handle network error on check out +description: checking in a connection after checkout fails closes connection runOn: - # required for appName in fail point @@ -29,27 +29,26 @@ operations: thread: thread1 # Wait for the check out to fail - name: waitForEvent - event: CheckOutFailed + event: ConnectionCheckOutFailed + count: 1 - name: checkIn connection: conn events: - type: ConnectionCheckOutStarted - address: 42 - type: ConnectionCreated - address: 42 - type: ConnectionReady - address: 42 - type: ConnectionCheckedOut - address: 42 - type: ConnectionCheckOutStarted - type: ConnectionCreated - type: ConnectionPoolCleared - type: ConnectionClosed reason: error - - type: CheckOutFailed + connectionId: 2 + - type: ConnectionCheckOutFailed reason: connectionError - type: ConnectionCheckedIn - type: ConnectionClosed + connectionId: 1 reason: stale ignore: - ConnectionPoolCreated diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json index ea7fe1e23f..cca12babe2 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json @@ -20,7 +20,8 @@ }, "poolOptions": { "maxPoolSize": 1, - "waitQueueTimeoutMS": 30000 + "waitQueueTimeoutMS": 30000, + "appName": "poolCheckOutErrorWaitQueueTest" }, "operations": [ { diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml index 481657e309..15f7e85036 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml @@ -14,6 +14,7 @@ failPoint: poolOptions: maxPoolSize: 1 waitQueueTimeoutMS: 30000 + appName: *appName operations: - name: ready # start a few threads that all will enter diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json index 43ecd2dc2f..ee04600d67 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json @@ -33,7 +33,7 @@ } ], "error": { - "type": "Network" + "type": "NetworkError" }, "events": [ { @@ -51,12 +51,12 @@ { "type": "ConnectionClosed", "address": 42, + "connectionId": 1, "reason": "error" }, { "type": "ConnectionCheckOutFailed", "address": 42, - "connectionId": 42, "reason": "connectionError" } ], diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml index 16d4ac61bf..43c8f44bc0 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml @@ -19,7 +19,7 @@ operations: - name: ready - name: checkOut error: - type: Network + type: NetworkError events: - type: ConnectionCheckOutStarted address: 42 @@ -29,10 +29,10 @@ events: address: 42 - type: ConnectionClosed address: 42 + connectionId: 1 reason: error - type: ConnectionCheckOutFailed address: 42 - connectionId: 42 reason: connectionError ignore: - ConnectionPoolCreated From a2e2c1e6c4eacb6d622b58fc2b630f6893fa9d8d Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 13 Jan 2023 16:35:17 -0500 Subject: [PATCH 04/24] add unified cmap tests --- .../tests/README.rst | 1 + .../tests/unified/auth-error.json | 172 ++++++++++ .../tests/unified/auth-error.yml | 100 ++++++ .../tests/unified/network-error.json | 316 ++++++++++++++++++ .../tests/unified/network-error.yml | 190 +++++++++++ 5 files changed, 779 insertions(+) create mode 100644 source/connection-monitoring-and-pooling/tests/unified/auth-error.json create mode 100644 source/connection-monitoring-and-pooling/tests/unified/auth-error.yml create mode 100644 source/connection-monitoring-and-pooling/tests/unified/network-error.json create mode 100644 source/connection-monitoring-and-pooling/tests/unified/network-error.yml diff --git a/source/connection-monitoring-and-pooling/tests/README.rst b/source/connection-monitoring-and-pooling/tests/README.rst index 08d5141494..75d60cf2b1 100644 --- a/source/connection-monitoring-and-pooling/tests/README.rst +++ b/source/connection-monitoring-and-pooling/tests/README.rst @@ -14,6 +14,7 @@ Introduction Drivers MUST implement all of the following types of CMAP tests: * Pool unit and integration tests as described in `cmap-format/README.rst <./cmap-format/README.rst>`__ +* `Unified Test Format <../../unified-test-format/unified-test-format.rst>`_ tests located in the ``unified`` directory * Pool prose tests as described below in `Prose Tests`_ * Logging tests as described below in `Logging Tests`_ diff --git a/source/connection-monitoring-and-pooling/tests/unified/auth-error.json b/source/connection-monitoring-and-pooling/tests/unified/auth-error.json new file mode 100644 index 0000000000..6704a54627 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/auth-error.json @@ -0,0 +1,172 @@ +{ + "description": "cmap-auth-error", + "schemaVersion": "1.10", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ], + "auth": true + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Pool is cleared after auth error during checkout", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "saslContinue" + ], + "errorCode": 18, + "appName": "cmapAuthErrorTest" + } + } + } + }, + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "connectionCheckOutStartedEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "connectionClosedEvent", + "poolClearedEvent", + "connectionReadyEvent", + "connectionCreatedEvent" + ], + "uriOptions": { + "retryWrites": false, + "retryReads": false, + "appname": "cmapAuthErrorTest" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "cmap-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "cmap-network-error" + } + } + ] + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionCheckOutFailedEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCreatedEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "error" + } + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/unified/auth-error.yml b/source/connection-monitoring-and-pooling/tests/unified/auth-error.yml new file mode 100644 index 0000000000..ce6edaf7bc --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/auth-error.yml @@ -0,0 +1,100 @@ +--- +description: cmap-auth-error + +schemaVersion: "1.10" + +runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single, replicaset, sharded ] + auth: true + +createEntities: + - client: + id: &setupClient setupClient + useMultipleMongoses: false + +initialData: &initialData + - collectionName: &collectionName cmap-network-error + databaseName: &databaseName cmap-tests + documents: + - _id: 1 + - _id: 2 + +tests: + - description: Pool is cleared after auth error during checkout + operations: + - name: failPoint + object: testRunner + arguments: + client: *setupClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - saslContinue + errorCode: 18 + appName: &authErrorAppName cmapAuthErrorTest + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: + - connectionCheckOutStartedEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + - connectionReadyEvent + - connectionCreatedEvent + uriOptions: + retryWrites: false + retryReads: false + appname: *authErrorAppName + - database: + id: &database database + client: *client + databaseName: *databaseName + - collection: + id: &collection collection + database: *database + collectionName: *collectionName + - name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + connectionCheckOutFailedEvent: {} + count: 1 + expectEvents: + - client: *client + eventType: cmap + events: + - connectionCheckOutStartedEvent: {} + - connectionCreatedEvent: {} + - poolClearedEvent: {} + - connectionClosedEvent: + reason: error + - connectionCheckOutFailedEvent: + reason: connectionError + + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - _id: 1 + - _id: 2 diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.json b/source/connection-monitoring-and-pooling/tests/unified/network-error.json new file mode 100644 index 0000000000..67a50bb99d --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.json @@ -0,0 +1,316 @@ +{ + "description": "cmap-network-error", + "schemaVersion": "1.10", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Pool is cleared after network error on checked out connection", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true, + "appName": "cmapNetworkErrorTest" + } + } + } + }, + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "connectionCheckOutStartedEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "connectionClosedEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "retryReads": false, + "appname": "cmapNetworkErrorTest" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "cmap-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "cmap-network-error" + } + } + ] + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionClosedEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "error" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "Pool is cleared after network error during checkout", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "connectionCheckOutStartedEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "connectionClosedEvent", + "poolClearedEvent", + "connectionCreatedEvent", + "connectionReadyEvent" + ], + "uriOptions": { + "retryWrites": false, + "retryReads": false, + "appName": "cmapNetworkErrorCheckOutTest", + "directConnection": true + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "cmap-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "cmap-network-error" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverDescriptionChangedEvent": { + "oldDescription": { + "type": "Unknown" + } + } + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 50 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "cmapNetworkErrorCheckOutTest" + } + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionCheckOutFailedEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCheckOutStartedEvent": {} + }, + { + "connectionCreatedEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "error" + } + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml new file mode 100644 index 0000000000..b851ae5572 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -0,0 +1,190 @@ +--- +description: cmap-network-error + +schemaVersion: "1.10" + +runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single, replicaset, sharded ] + +createEntities: + - client: + id: &setupClient setupClient + useMultipleMongoses: false + +initialData: &initialData + - collectionName: &collectionName cmap-network-error + databaseName: &databaseName cmap-tests + documents: + - _id: 1 + - _id: 2 + +tests: + - description: Pool is cleared after network error on checked out connection + operations: + - name: failPoint + object: testRunner + arguments: + client: *setupClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - find + closeConnection: true + appName: &appName cmapNetworkErrorTest + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: + - connectionCheckOutStartedEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + uriOptions: + retryWrites: false + retryReads: false + appname: *appName + - database: + id: &database database + client: *client + databaseName: *databaseName + - collection: + id: &collection collection + database: *database + collectionName: *collectionName + - name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + connectionClosedEvent: {} + count: 1 + expectEvents: + - client: *client + eventType: cmap + events: + - connectionCheckOutStartedEvent: {} + - connectionCheckedOutEvent: {} + # SDAM error handling needs to be performed before the connection is checked into the pool. + - poolClearedEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: + # connectionId: 1 + # Even though the connection is also stale, the reason it is being closed + # is due to an error. + reason: error + + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - _id: 1 + - _id: 2 + + - description: Pool is cleared after network error during checkout + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: + - connectionCheckOutStartedEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + - connectionCreatedEvent + - connectionReadyEvent + uriOptions: + retryWrites: false + retryReads: false + appName: &networkErrorCheckoutAppName cmapNetworkErrorCheckOutTest + directConnection: true + - database: + id: &database database + client: *client + databaseName: *databaseName + - collection: + id: &collection collection + database: *database + collectionName: *collectionName + # wait for the server to be discovered before enabling the failpoint + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + serverDescriptionChangedEvent: + oldDescription: + type: Unknown + count: 1 + - name: failPoint + object: testRunner + arguments: + client: *setupClient + failPoint: + configureFailPoint: failCommand + mode: + # use a high number of times to prevent monitor hellos from interfering. + times: 50 + data: + failCommands: + - isMaster + - hello + closeConnection: true + appName: *networkErrorCheckoutAppName + - name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + connectionCheckOutFailedEvent: {} + count: 1 + expectEvents: + - client: *client + eventType: cmap + events: + - connectionCheckOutStartedEvent: {} + - connectionCreatedEvent: {} + # SDAM needs to handle the error before the pool proceeds with cleaning up the connection. + - poolClearedEvent: {} + - connectionClosedEvent: + reason: error + - connectionCheckOutFailedEvent: + reason: connectionError + + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - _id: 1 + - _id: 2 From 143963c8cb2760c227537ced9d648177a4dac6aa Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 19 Jan 2023 16:47:09 -0500 Subject: [PATCH 05/24] test finishes --- .../tests/unified/network-error.json | 192 ++++++++++++++++-- .../tests/unified/network-error.yml | 113 ++++++++++- 2 files changed, 284 insertions(+), 21 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.json b/source/connection-monitoring-and-pooling/tests/unified/network-error.json index 67a50bb99d..5207630c98 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.json +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.json @@ -1,17 +1,6 @@ { "description": "cmap-network-error", "schemaVersion": "1.10", - "runOnRequirements": [ - { - "minServerVersion": "4.4", - "serverless": "forbid", - "topologies": [ - "single", - "replicaset", - "sharded" - ] - } - ], "createEntities": [ { "client": { @@ -37,6 +26,17 @@ "tests": [ { "description": "Pool is cleared after network error on checked out connection", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], "operations": [ { "name": "failPoint", @@ -165,6 +165,15 @@ }, { "description": "Pool is cleared after network error during checkout", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single" + ] + } + ], "operations": [ { "name": "createEntities", @@ -188,8 +197,7 @@ "uriOptions": { "retryWrites": false, "retryReads": false, - "appName": "cmapNetworkErrorCheckOutTest", - "directConnection": true + "appName": "cmapNetworkErrorCheckOutTest" } } }, @@ -219,6 +227,9 @@ "serverDescriptionChangedEvent": { "oldDescription": { "type": "Unknown" + }, + "newDescription": { + "type": "Standalone" } } }, @@ -311,6 +322,161 @@ ] } ] + }, + { + "description": "Network error on minPoolSize background creation", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ], + "auth": true + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "saslContinue" + ], + "appName": "CMAPminPoolSizeError", + "closeConnection": true + } + } + } + }, + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "poolClearedEvent", + "poolReadyEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent" + ], + "uriOptions": { + "heartbeatFrequencyMS": 500, + "appname": "CMAPminPoolSizeError", + "minPoolSize": 1, + "serverSelectionTimeoutMS": 1000 + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "cmap-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "cmap-network-error" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolReadyEvent": {} + }, + "count": 1 + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionClosedEvent": { + "reason": "error" + } + }, + "count": 1 + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolReadyEvent": {} + }, + "count": 2 + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionReadyEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "poolReadyEvent": {} + }, + { + "connectionCreatedEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "error" + } + }, + { + "poolReadyEvent": {} + }, + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + } + ] + } + ] } ] } diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml index b851ae5572..754ecf2a6a 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -3,12 +3,6 @@ description: cmap-network-error schemaVersion: "1.10" -runOnRequirements: - # failCommand appName requirements - - minServerVersion: "4.4" - serverless: forbid - topologies: [ single, replicaset, sharded ] - createEntities: - client: id: &setupClient setupClient @@ -23,6 +17,11 @@ initialData: &initialData tests: - description: Pool is cleared after network error on checked out connection + runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single, replicaset, sharded ] operations: - name: failPoint object: testRunner @@ -87,7 +86,6 @@ tests: - poolClearedEvent: {} - connectionCheckedInEvent: {} - connectionClosedEvent: - # connectionId: 1 # Even though the connection is also stale, the reason it is being closed # is due to an error. reason: error @@ -100,6 +98,11 @@ tests: - _id: 2 - description: Pool is cleared after network error during checkout + runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single ] operations: - name: createEntities object: testRunner @@ -121,7 +124,6 @@ tests: retryWrites: false retryReads: false appName: &networkErrorCheckoutAppName cmapNetworkErrorCheckOutTest - directConnection: true - database: id: &database database client: *client @@ -139,6 +141,8 @@ tests: serverDescriptionChangedEvent: oldDescription: type: Unknown + newDescription: + type: Standalone count: 1 - name: failPoint object: testRunner @@ -188,3 +192,96 @@ tests: documents: - _id: 1 - _id: 2 + + - description: Network error on minPoolSize background creation + runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single, replicaset, sharded ] + auth: true + operations: + # Configure the initial monitor handshake to succeed but the + # first background minPoolSize establishment to fail. + - name: failPoint + object: testRunner + arguments: + client: *setupClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - saslContinue + appName: &minPoolSizeAppName CMAPminPoolSizeError + closeConnection: true + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: + - poolClearedEvent + - poolReadyEvent + - connectionCreatedEvent + - connectionReadyEvent + - connectionClosedEvent + - connectionCheckOutStartedEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + uriOptions: + heartbeatFrequencyMS: 500 + appname: *minPoolSizeAppName + minPoolSize: 1 + # Wait for monitor to succeed handshake and mark the pool as ready. + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + poolReadyEvent: {} + count: 1 + # Background connection establishment due to minPoolSize should fail, + # causing the connection to close. + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + connectionClosedEvent: + reason: error + count: 1 + # Wait for monitor to rediscover server and mark pool as ready again. + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + poolReadyEvent: {} + count: 2 + # Now that the failpoint has been disabled (was set to times: 1), background + # connection creation should succeed. + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + connectionReadyEvent: {} + count: 1 + + expectEvents: + - client: *client + eventType: cmap + events: + - poolReadyEvent: {} + - connectionCreatedEvent: {} + - poolClearedEvent: {} + - connectionClosedEvent: + reason: error + - poolReadyEvent: {} + - connectionCreatedEvent: {} + - connectionReadyEvent: {} From 91876342de0d45e3369164cd5b490c7b08ace5cd Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 19 Jan 2023 16:47:16 -0500 Subject: [PATCH 06/24] error handling prose --- .../connection-monitoring-and-pooling.rst | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst index 0fe3811ae7..481a25f9ac 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst @@ -452,6 +452,11 @@ Before a `Connection <#connection>`_ can be marked as either "available" or "in must be established. This process involves performing the initial handshake, handling OP_COMPRESSED, and performing authentication. +If an error is encountered while attempting to establish a connection, it MUST be handled according +to the `Application Errors`_ section in the SDAM specification. After the SDAM machinery has handled the error, +the connection MUST be `closed <#closing-a-connection-internal-implementation>`_. The error can then be propagated +to the context that initiated the connection establishment attempt. + .. code:: try: @@ -462,8 +467,11 @@ handshake, handling OP_COMPRESSED, and performing authentication. emit ConnectionReadyEvent and equivalent log message return connection except error: + # Handle error according to SDAM. This may entail clearing the pool. + topology.handle_pre_handshake_error(error) close connection - throw error # Propagate error in manner idiomatic to language. + # Propagate error in manner idiomatic to language. + throw error Closing a Connection (Internal Implementation) @@ -519,10 +527,6 @@ Populating the pool MUST NOT block any application threads. For example, it could be performed on a background thread or via the use of non-blocking/async I/O. Populating the pool MUST NOT be performed unless the pool is "ready". -If an error is encountered while populating a connection, it MUST be handled via -the SDAM machinery according to the `Application Errors`_ section in the SDAM -specification. - If minPoolSize is set, the `Connection <#connection>`_ Pool MUST be populated until it has at least minPoolSize total `Connections <#connection>`_. This MUST occur only while the pool is "ready". If the pool implements a background @@ -544,12 +548,8 @@ established as a result of populating the Pool. wait until pendingConnectionCount < maxConnecting and pool is "ready" create connection - try: - establish connection - mark connection as available - except error: - # Defer error handling to SDAM. - topology.handle_pre_handshake_error(error) + try establish connection + mark connection as available Checking Out a Connection ------------------------- @@ -656,10 +656,7 @@ Before a given `Connection <#connection>`_ is returned from checkOut, it must be establish connection except connection establishment error: emit ConnectionCheckOutFailedEvent(reason="connectionError") and equivalent log message - decrement totalConnectionCount throw - finally: - decrement pendingConnectionCount else: decrement availableConnectionCount set connection state to "in use" @@ -683,6 +680,9 @@ true: Otherwise, the `Connection <#connection>`_ is marked as available. +If an error is encountered while reading from or writing to a `Connection <#connection>`_, the error MUST be handled +according to the `Application Errors`_ section of the SDAM specification *before* the connection is checked back into the pool. + .. code:: emit ConnectionCheckedInEvent and equivalent log message From 423cca665ae7c2a977656c66cc345881f1461527 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 19 Jan 2023 16:47:23 -0500 Subject: [PATCH 07/24] error type clarification --- .../tests/cmap-format/README.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst b/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst index 229a30bf0d..a29649c78e 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst @@ -54,7 +54,14 @@ All Unit Tests have some of the following fields: - ``error``: Indicates that the main thread is expected to error during this test. An error may include of the following fields: - - ``type``: the type of error emitted + - ``type``: the type of error emitted. Currently, only the following error types are supported: + + - WaitQueueTimeoutError + - PoolClosedError + - PoolClearedError + - AuthenticationError + - NetworkError + - ``message``: the message associated with that error - ``address``: Address of pool emitting error From d31abde021ff38a4fbba9fc8026ad59e5ee54892 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 19 Jan 2023 17:04:07 -0500 Subject: [PATCH 08/24] some touches --- .../connection-monitoring-and-pooling.rst | 5 ++-- .../tests/unified/network-error.json | 29 ++++++------------- .../tests/unified/network-error.yml | 6 ++-- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst index 481a25f9ac..e8fd4f8bfa 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst @@ -680,8 +680,9 @@ true: Otherwise, the `Connection <#connection>`_ is marked as available. -If an error is encountered while reading from or writing to a `Connection <#connection>`_, the error MUST be handled -according to the `Application Errors`_ section of the SDAM specification *before* the connection is checked back into the pool. +If an error is encountered while reading from or writing to a checked out `Connection <#connection>`_, the error MUST +be handled according to the `Application Errors`_ section of the SDAM specification *before* the connection is checked +back into the pool. .. code:: diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.json b/source/connection-monitoring-and-pooling/tests/unified/network-error.json index 5207630c98..9c19ee18e5 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.json +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.json @@ -25,7 +25,7 @@ ], "tests": [ { - "description": "Pool is cleared after network error on checked out connection", + "description": "Pool properly handles network error on checked out connection", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -164,7 +164,7 @@ ] }, { - "description": "Pool is cleared after network error during checkout", + "description": "Pool properly handles network error during checkout", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -324,7 +324,7 @@ ] }, { - "description": "Network error on minPoolSize background creation", + "description": "Pool properly handles network error during minPoolSize background connection creation", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -372,29 +372,18 @@ "poolReadyEvent", "connectionCreatedEvent", "connectionReadyEvent", - "connectionClosedEvent" + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent" ], "uriOptions": { "heartbeatFrequencyMS": 500, "appname": "CMAPminPoolSizeError", - "minPoolSize": 1, - "serverSelectionTimeoutMS": 1000 + "minPoolSize": 1 } } - }, - { - "database": { - "id": "database", - "client": "client", - "databaseName": "cmap-tests" - } - }, - { - "collection": { - "id": "collection", - "database": "database", - "collectionName": "cmap-network-error" - } } ] } diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml index 754ecf2a6a..f1c13695ff 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -16,7 +16,7 @@ initialData: &initialData - _id: 2 tests: - - description: Pool is cleared after network error on checked out connection + - description: Pool properly handles network error on checked out connection runOnRequirements: # failCommand appName requirements - minServerVersion: "4.4" @@ -97,7 +97,7 @@ tests: - _id: 1 - _id: 2 - - description: Pool is cleared after network error during checkout + - description: Pool properly handles network error during checkout runOnRequirements: # failCommand appName requirements - minServerVersion: "4.4" @@ -193,7 +193,7 @@ tests: - _id: 1 - _id: 2 - - description: Network error on minPoolSize background creation + - description: Pool properly handles network error during minPoolSize background connection creation runOnRequirements: # failCommand appName requirements - minServerVersion: "4.4" From 72f5bbdcfc0dd3f28e17cef53ef3ab28488ffd23 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 19 Jan 2023 17:14:44 -0500 Subject: [PATCH 09/24] eliminate failpoint cmap format operation? --- .../pool-checkout-error-checkin.json | 33 +++++++++---------- .../pool-checkout-error-checkin.yml | 22 ++++++------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json index cda624ba59..51a9cc18b1 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.json @@ -4,12 +4,26 @@ "description": "checking in a connection after checkout fails closes connection", "runOn": [ { - "minServerVersion": "4.9.0" + "minServerVersion": "4.9.0", + "auth": true } ], "poolOptions": { "appName": "poolCheckOutErrorCheckInTest" }, + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "skip": 1 + }, + "data": { + "failCommands": [ + "saslContinue" + ], + "closeConnection": true, + "appName": "poolCheckOutErrorCheckInTest" + } + }, "operations": [ { "name": "ready" @@ -18,23 +32,6 @@ "name": "checkOut", "label": "conn" }, - { - "name": "failPoint", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 10 - }, - "data": { - "failCommands": [ - "isMaster", - "hello" - ], - "closeConnection": true, - "appName": "poolCheckOutErrorCheckInTest" - } - } - }, { "name": "start", "target": "thread1" diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml index a2d2a47e50..28bea02730 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-checkin.yml @@ -3,24 +3,24 @@ style: integration description: checking in a connection after checkout fails closes connection runOn: - - # required for appName in fail point + # Required for appName in fail point minServerVersion: "4.9.0" + auth: true poolOptions: appName: &appName "poolCheckOutErrorCheckInTest" +failPoint: + configureFailPoint: failCommand + # Skip the first so that the first checkOut succeeds. + mode: { skip: 1 } + data: + # Use saslContinue so that monitor checks won't interfere with this failpoint. + failCommands: ["saslContinue"] + closeConnection: true + appName: *appName operations: - name: ready - name: checkOut label: conn - # Enable a failpoint so the next checkOut will fail. - - name: failPoint - failPoint: - configureFailPoint: failCommand - # Use a high number of times to prevent monitor checks from interfering - mode: { times: 10 } - data: - failCommands: ["isMaster","hello"] - closeConnection: true - appName: *appName # Perform check out in another thread so the main thread can continue to # execute operations. - name: start From 471b39fecd736533e1df455fc815c22c265bff35 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 14:18:48 -0500 Subject: [PATCH 10/24] fix connection id in auth test --- .../tests/cmap-format/pool-checkout-error-auth.json | 7 ++++--- .../tests/cmap-format/pool-checkout-error-auth.yml | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json index edd22728c1..208e8653e6 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json @@ -33,8 +33,7 @@ } ], "error": { - "type": "AuthenticationError", - "message": "????????" + "type": "AuthenticationError" }, "events": [ { @@ -43,6 +42,7 @@ }, { "type": "ConnectionCreated", + "connectionId": 1, "address": 42 }, { @@ -51,13 +51,14 @@ }, { "type": "ConnectionClosed", + "connectionId": 1, "address": 42, "reason": "error" }, { "type": "ConnectionCheckOutFailed", + "connectionId": 1, "address": 42, - "connectionId": 42, "reason": "connectionError" } ], diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml index d9cc49ce32..6ae427122e 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml @@ -24,15 +24,17 @@ events: - type: ConnectionCheckOutStarted address: 42 - type: ConnectionCreated + connectionId: 1 address: 42 - type: ConnectionPoolCleared address: 42 - type: ConnectionClosed + connectionId: 1 address: 42 reason: error - type: ConnectionCheckOutFailed + connectionId: 1 address: 42 - connectionId: 42 reason: connectionError ignore: - ConnectionPoolCreated From 88426d03d468c36fa5b9417665e84517afd9dd01 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 14:22:11 -0500 Subject: [PATCH 11/24] remove failpoint operation from CMAP format --- .../tests/cmap-format/README.rst | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst b/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst index a29649c78e..734f80f094 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/README.rst @@ -142,23 +142,6 @@ the addition of the following fields to each test: - ``poolOptions.appName`` (optional): appName attribute to be set in connections, which will be affected by the fail point. -The integration also supports the following additional operations which are not available -in the unit test format: - -failPoint -~~~~~~~~~ - -The ``failPoint`` operation instructs the test runner to configure a fail point -on the endpoint being used for the test. - -The following arguments are supported: - -- ``failPoint``: Required document. The ``configureFailPoint`` command to be - executed. - -When executing this operation, the test runner MUST keep a record of the fail -point so that it can be disabled after the test. - Spec Test Match Function ======================== From 91b4d9c61c549ff68adceb67ebd34e964e626e69 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 15:10:00 -0500 Subject: [PATCH 12/24] don't ignore events test doesnt expect to see --- .../tests/cmap-format/pool-checkout-error-clears-waitqueue.json | 2 -- .../tests/cmap-format/pool-checkout-error-clears-waitqueue.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json index cca12babe2..8fef127e7d 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json @@ -100,8 +100,6 @@ "ConnectionPoolReady", "ConnectionCreated", "ConnectionPoolCreated", - "ConnectionReady", - "ConnectionCheckedIn", "ConnectionClosed" ] } diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml index 15f7e85036..b2da320baa 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml @@ -61,6 +61,4 @@ ignore: - ConnectionPoolReady - ConnectionCreated - ConnectionPoolCreated - - ConnectionReady - - ConnectionCheckedIn - ConnectionClosed From 55bf35a86936aa8993059b1231559ea90d7bd9b7 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 15:16:07 -0500 Subject: [PATCH 13/24] add concurrent error test --- .../pool-checkout-error-concurrent.json | 82 +++++++++++++++++++ .../pool-checkout-error-concurrent.yml | 47 +++++++++++ 2 files changed, 129 insertions(+) create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json new file mode 100644 index 0000000000..c69c5580ee --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json @@ -0,0 +1,82 @@ +{ + "version": 1, + "style": "integration", + "description": "must properly handle multiple concurrent network errors on check out", + "runOn": [ + { + "minServerVersion": "4.9.0" + } + ], + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 50 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "poolCheckOutErrorTest" + } + }, + "poolOptions": { + "appName": "poolCheckOutErrorTest" + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "start", + "target": "thread1" + }, + { + "name": "start", + "target": "thread2" + }, + { + "name": "start", + "target": "thread3" + }, + { + "name": "checkOut", + "thread": "thread1" + }, + { + "name": "checkOut", + "thread": "thread2" + }, + { + "name": "checkOut", + "thread": "thread3" + }, + { + "name": "waitForEvent", + "event": "ConnectionCheckOutFailed", + "count": 3 + }, + { + "name": "ready" + } + ], + "events": [ + { + "type": "ConnectionPoolReady" + }, + { + "type": "ConnectionPoolCleared" + }, + { + "type": "ConnectionPoolReady" + } + ], + "ignore": [ + "ConnectionCheckOutStarted", + "ConnectionCheckOutFailed", + "ConnectionPoolCreated", + "ConnectionCreated", + "ConnectionClosed" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml new file mode 100644 index 0000000000..12dcfb2135 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml @@ -0,0 +1,47 @@ +version: 1 +style: integration +description: must properly handle multiple concurrent network errors on check out +runOn: + - + # required for appName in fail point + minServerVersion: "4.9.0" +failPoint: + configureFailPoint: failCommand + # high amount to ensure not interfered with by monitor checks. + mode: { times: 50 } + data: + failCommands: ["isMaster","hello"] + closeConnection: true + appName: &appName "poolCheckOutErrorTest" +poolOptions: + appName: *appName +operations: + - name: ready + - name: start + target: thread1 + - name: start + target: thread2 + - name: start + target: thread3 + - name: checkOut + thread: thread1 + - name: checkOut + thread: thread2 + - name: checkOut + thread: thread3 + - name: waitForEvent + event: ConnectionCheckOutFailed + count: 3 + - name: ready +events: + - type: ConnectionPoolReady + # Pool should only be cleared once. + - type: ConnectionPoolCleared + - type: ConnectionPoolReady +ignore: + - ConnectionCheckOutStarted + - ConnectionCheckOutFailed + - ConnectionPoolCreated + - ConnectionCreated + - ConnectionClosed + From 0be3db69714479b9eec223f5701aa9db89f2ed4a Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:04:03 -0500 Subject: [PATCH 14/24] combine logging unified tests into one unified directory --- source/connection-monitoring-and-pooling/tests/README.rst | 8 +------- .../tests/{logging => unified}/connection-logging.json | 0 .../tests/{logging => unified}/connection-logging.yml | 0 .../connection-pool-options-logging.json} | 0 .../connection-pool-options-logging.yml} | 0 5 files changed, 1 insertion(+), 7 deletions(-) rename source/connection-monitoring-and-pooling/tests/{logging => unified}/connection-logging.json (100%) rename source/connection-monitoring-and-pooling/tests/{logging => unified}/connection-logging.yml (100%) rename source/connection-monitoring-and-pooling/tests/{logging/connection-pool-options.json => unified/connection-pool-options-logging.json} (100%) rename source/connection-monitoring-and-pooling/tests/{logging/connection-pool-options.yml => unified/connection-pool-options-logging.yml} (100%) diff --git a/source/connection-monitoring-and-pooling/tests/README.rst b/source/connection-monitoring-and-pooling/tests/README.rst index 75d60cf2b1..ae56bcdd5d 100644 --- a/source/connection-monitoring-and-pooling/tests/README.rst +++ b/source/connection-monitoring-and-pooling/tests/README.rst @@ -14,9 +14,8 @@ Introduction Drivers MUST implement all of the following types of CMAP tests: * Pool unit and integration tests as described in `cmap-format/README.rst <./cmap-format/README.rst>`__ -* `Unified Test Format <../../unified-test-format/unified-test-format.rst>`_ tests located in the ``unified`` directory +* Integration tests written in the `Unified Test Format <../../unified-test-format/unified-test-format.rst>`_ located in the `/unified <./unified>` directory * Pool prose tests as described below in `Prose Tests`_ -* Logging tests as described below in `Logging Tests`_ Prose Tests =========== @@ -28,8 +27,3 @@ The following tests have not yet been automated, but MUST still be tested: #. A user MUST be able to specify all ConnectionPoolOptions via a URI string #. A user MUST be able to subscribe to Connection Monitoring Events in a manner idiomatic to their language and driver -Logging Tests -============= - -Tests for connection pool logging can be found in the `/logging <./logging>`__ subdirectory and are written in the -`Unified Test Format <../../unified-test-format/unified-test-format.rst>`__. diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/unified/connection-logging.json similarity index 100% rename from source/connection-monitoring-and-pooling/tests/logging/connection-logging.json rename to source/connection-monitoring-and-pooling/tests/unified/connection-logging.json diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml b/source/connection-monitoring-and-pooling/tests/unified/connection-logging.yml similarity index 100% rename from source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml rename to source/connection-monitoring-and-pooling/tests/unified/connection-logging.yml diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-pool-options.json b/source/connection-monitoring-and-pooling/tests/unified/connection-pool-options-logging.json similarity index 100% rename from source/connection-monitoring-and-pooling/tests/logging/connection-pool-options.json rename to source/connection-monitoring-and-pooling/tests/unified/connection-pool-options-logging.json diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-pool-options.yml b/source/connection-monitoring-and-pooling/tests/unified/connection-pool-options-logging.yml similarity index 100% rename from source/connection-monitoring-and-pooling/tests/logging/connection-pool-options.yml rename to source/connection-monitoring-and-pooling/tests/unified/connection-pool-options-logging.yml From 7f9ad155f4c708b3ad54c12c6997408bb6bef9ca Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:04:52 -0500 Subject: [PATCH 15/24] update checkout prose to mention errors, properly handle pendingConnectionCount in psuedocode --- .../connection-monitoring-and-pooling.rst | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst index e8fd4f8bfa..e580cc5115 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst @@ -550,6 +550,7 @@ established as a result of populating the Pool. create connection try establish connection mark connection as available + decrement pendingConnectionCount Checking Out a Connection ------------------------- @@ -573,17 +574,26 @@ available `Connection`_ is found or the list of available `Connections If the list is exhausted, the total number of `Connections <#connection>`_ is less than maxPoolSize, and pendingConnectionCount < maxConnecting, the pool MUST -create a `Connection`_, establish it, mark it as "in use" and return it. If -totalConnectionCount == maxPoolSize or pendingConnectionCount == maxConnecting, -then the pool MUST wait to service the request until neither of those two -conditions are met or until a `Connection`_ becomes available, re-entering the -checkOut loop in either case. This waiting MUST NOT prevent `Connections -<#connection>`_ from being checked into the pool. Additionally, the Pool MUST -NOT service any newer checkOut requests before fulfilling the original one which -could not be fulfilled. For drivers that implement the WaitQueue via a fair -semaphore, a condition variable may also be needed to to meet this -requirement. Waiting on the condition variable SHOULD also be limited by the -WaitQueueTimeout, if the driver supports one and it was specified by the user. +create a `Connection`_ (as described in `Creating a Connection +<#creating-a-connection-internal-implementation>`_), establish it (as described +in `Establishing a Connection +<#establishing-a-connection-internal-implementation>`_), mark it as "in use" and +return it. If an error is encountered while attempting to establish the +connection, the pool MUST emit a ConnectionCheckOutFailed event with reason +"connectionError" and corresponding log message before propagating the error to +connection requester. + +If the list is exhausted and totalConnectionCount == maxPoolSize or +pendingConnectionCount == maxConnecting, then the pool MUST wait to service the +request until neither of those two conditions are met or until a `Connection`_ +becomes available, re-entering the checkOut loop in either case. This waiting +MUST NOT prevent `Connections <#connection>`_ from being checked into the +pool. Additionally, the Pool MUST NOT service any newer checkOut requests before +fulfilling the original one which could not be fulfilled. For drivers that +implement the WaitQueue via a fair semaphore, a condition variable may also be +needed to to meet this requirement. Waiting on the condition variable SHOULD +also be limited by the WaitQueueTimeout, if the driver supports one and it was +specified by the user. If the pool is "closed" or "paused", any attempt to check out a `Connection <#connection>`_ MUST throw an Error. The error thrown as a result of the pool @@ -654,12 +664,14 @@ Before a given `Connection <#connection>`_ is returned from checkOut, it must be if connection state is "pending": try: establish connection + set connection state to "in use" + decrement pendingConnectionCount except connection establishment error: emit ConnectionCheckOutFailedEvent(reason="connectionError") and equivalent log message throw else: + set connection state to "in use" decrement availableConnectionCount - set connection state to "in use" emit ConnectionCheckedOutEvent and equivalent log message return connection From 81e3f944ba1830509e3fa08fe5c585c542b10b4a Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:12:50 -0500 Subject: [PATCH 16/24] fix typos --- .../connection-monitoring-and-pooling.rst | 4 ++-- source/connection-monitoring-and-pooling/tests/README.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst index e580cc5115..ff0e440603 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst @@ -580,8 +580,8 @@ in `Establishing a Connection <#establishing-a-connection-internal-implementation>`_), mark it as "in use" and return it. If an error is encountered while attempting to establish the connection, the pool MUST emit a ConnectionCheckOutFailed event with reason -"connectionError" and corresponding log message before propagating the error to -connection requester. +"connectionError" and a corresponding log message before propagating the error +to connection requester. If the list is exhausted and totalConnectionCount == maxPoolSize or pendingConnectionCount == maxConnecting, then the pool MUST wait to service the diff --git a/source/connection-monitoring-and-pooling/tests/README.rst b/source/connection-monitoring-and-pooling/tests/README.rst index ae56bcdd5d..e9c71630d5 100644 --- a/source/connection-monitoring-and-pooling/tests/README.rst +++ b/source/connection-monitoring-and-pooling/tests/README.rst @@ -14,7 +14,7 @@ Introduction Drivers MUST implement all of the following types of CMAP tests: * Pool unit and integration tests as described in `cmap-format/README.rst <./cmap-format/README.rst>`__ -* Integration tests written in the `Unified Test Format <../../unified-test-format/unified-test-format.rst>`_ located in the `/unified <./unified>` directory +* Integration tests written in the `Unified Test Format <../../unified-test-format/unified-test-format.rst>`_ located in the `/unified <./unified>`_ directory * Pool prose tests as described below in `Prose Tests`_ Prose Tests From ccee026bffc951c3d1f9d7a42759c67c85e4586e Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:26:00 -0500 Subject: [PATCH 17/24] update changelog --- .../connection-monitoring-and-pooling.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst index ff0e440603..05622ec713 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst @@ -1560,7 +1560,8 @@ Changelog :2021-11-08: Make maxConnecting configurable. :2022-04-05: Preemptively cancel in progress operations when SDAM heartbeats timeout. :2022-10-05: Remove spec front matter and reformat changelog. -:2022-10-14: Add connection pool log messages and associated tests. +:2022-10-14: Add connection pool log messages and associated tests +:2022-01-20: Clarify error handling semantics and add associated tests. ---- From d721af2e25423269db06bcaa73d606aacd9b5807 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:28:40 -0500 Subject: [PATCH 18/24] add period back in --- .../connection-monitoring-and-pooling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst index 05622ec713..5963befd64 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.rst @@ -1560,7 +1560,7 @@ Changelog :2021-11-08: Make maxConnecting configurable. :2022-04-05: Preemptively cancel in progress operations when SDAM heartbeats timeout. :2022-10-05: Remove spec front matter and reformat changelog. -:2022-10-14: Add connection pool log messages and associated tests +:2022-10-14: Add connection pool log messages and associated tests. :2022-01-20: Clarify error handling semantics and add associated tests. ---- From 4d28ee3e53fee5b931cf272dff72526c0dd513c9 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:55:05 -0500 Subject: [PATCH 19/24] skip tests on replicasets that need a single server --- .../tests/unified/network-error.json | 13 +++---------- .../tests/unified/network-error.yml | 10 +++------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.json b/source/connection-monitoring-and-pooling/tests/unified/network-error.json index 9c19ee18e5..6925f0a791 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.json +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.json @@ -170,7 +170,8 @@ "minServerVersion": "4.4", "serverless": "forbid", "topologies": [ - "single" + "single", + "sharded" ] } ], @@ -224,14 +225,7 @@ "arguments": { "client": "client", "event": { - "serverDescriptionChangedEvent": { - "oldDescription": { - "type": "Unknown" - }, - "newDescription": { - "type": "Standalone" - } - } + "poolReadyEvent": {} }, "count": 1 } @@ -331,7 +325,6 @@ "serverless": "forbid", "topologies": [ "single", - "replicaset", "sharded" ], "auth": true diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml index f1c13695ff..ff8f79ad9e 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -102,7 +102,7 @@ tests: # failCommand appName requirements - minServerVersion: "4.4" serverless: forbid - topologies: [ single ] + topologies: [ single, sharded ] operations: - name: createEntities object: testRunner @@ -138,11 +138,7 @@ tests: arguments: client: *client event: - serverDescriptionChangedEvent: - oldDescription: - type: Unknown - newDescription: - type: Standalone + poolReadyEvent: {} count: 1 - name: failPoint object: testRunner @@ -198,7 +194,7 @@ tests: # failCommand appName requirements - minServerVersion: "4.4" serverless: forbid - topologies: [ single, replicaset, sharded ] + topologies: [ single, sharded ] auth: true operations: # Configure the initial monitor handshake to succeed but the From b1c7f02a7ec97dc9ffdfbd43ad1285991e675e5f Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 20 Jan 2023 16:57:40 -0500 Subject: [PATCH 20/24] fix appname --- .../tests/unified/network-error.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml index ff8f79ad9e..147f64597e 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -123,7 +123,7 @@ tests: uriOptions: retryWrites: false retryReads: false - appName: &networkErrorCheckoutAppName cmapNetworkErrorCheckOutTest + appName: &networkErrorCheckOutAppName cmapNetworkErrorCheckOutTest - database: id: &database database client: *client @@ -154,7 +154,7 @@ tests: - isMaster - hello closeConnection: true - appName: *networkErrorCheckoutAppName + appName: *networkErrorCheckOutAppName - name: find object: *collection arguments: From dbb8b671c6dc7d2b8766164eb3c81d96f8928c6a Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 2 Feb 2023 18:14:33 -0500 Subject: [PATCH 21/24] add poolreadyevent to observe events --- .../tests/unified/network-error.json | 6 +++++- .../tests/unified/network-error.yml | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.json b/source/connection-monitoring-and-pooling/tests/unified/network-error.json index 6925f0a791..b2b62daa2b 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.json +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.json @@ -193,7 +193,8 @@ "connectionClosedEvent", "poolClearedEvent", "connectionCreatedEvent", - "connectionReadyEvent" + "connectionReadyEvent", + "poolReadyEvent" ], "uriOptions": { "retryWrites": false, @@ -280,6 +281,9 @@ "client": "client", "eventType": "cmap", "events": [ + { + "poolReadyEvent": {} + }, { "connectionCheckOutStartedEvent": {} }, diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml index 147f64597e..62c2f57fdc 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -120,6 +120,7 @@ tests: - poolClearedEvent - connectionCreatedEvent - connectionReadyEvent + - poolReadyEvent uriOptions: retryWrites: false retryReads: false @@ -173,6 +174,7 @@ tests: - client: *client eventType: cmap events: + - poolReadyEvent: {} - connectionCheckOutStartedEvent: {} - connectionCreatedEvent: {} # SDAM needs to handle the error before the pool proceeds with cleaning up the connection. From aa0d7ae02ae80be44177c238742179d00a0090a9 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 2 Feb 2023 18:14:52 -0500 Subject: [PATCH 22/24] fix race in test --- .../tests/unified/network-error.json | 5 +++-- .../tests/unified/network-error.yml | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.json b/source/connection-monitoring-and-pooling/tests/unified/network-error.json index b2b62daa2b..2db0396e25 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.json +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.json @@ -199,7 +199,8 @@ "uriOptions": { "retryWrites": false, "retryReads": false, - "appName": "cmapNetworkErrorCheckOutTest" + "appName": "cmapNetworkErrorCheckOutTest", + "heartbeatFrequencyMS": 30000 } } }, @@ -239,7 +240,7 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 50 + "times": 1 }, "data": { "failCommands": [ diff --git a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml index 62c2f57fdc..3a42ecbfd4 100644 --- a/source/connection-monitoring-and-pooling/tests/unified/network-error.yml +++ b/source/connection-monitoring-and-pooling/tests/unified/network-error.yml @@ -125,6 +125,9 @@ tests: retryWrites: false retryReads: false appName: &networkErrorCheckOutAppName cmapNetworkErrorCheckOutTest + # Use a high frequency so that server does not get marked unknown before the find + # can be executed. + heartbeatFrequencyMS: 30000 - database: id: &database database client: *client @@ -148,8 +151,7 @@ tests: failPoint: configureFailPoint: failCommand mode: - # use a high number of times to prevent monitor hellos from interfering. - times: 50 + times: 1 data: failCommands: - isMaster From bec7969f068b7e1778bd6929d4ae9a7157e2b749 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Thu, 2 Feb 2023 18:16:20 -0500 Subject: [PATCH 23/24] remove extra connectionId field --- .../tests/cmap-format/pool-checkout-error-auth.json | 1 - .../tests/cmap-format/pool-checkout-error-auth.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json index 208e8653e6..328da95c24 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json @@ -57,7 +57,6 @@ }, { "type": "ConnectionCheckOutFailed", - "connectionId": 1, "address": 42, "reason": "connectionError" } diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml index 6ae427122e..9aecc785df 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml @@ -33,7 +33,6 @@ events: address: 42 reason: error - type: ConnectionCheckOutFailed - connectionId: 1 address: 42 reason: connectionError ignore: From 3c0d6a3cb5b40b71c875690ff55b822509a666f2 Mon Sep 17 00:00:00 2001 From: Patrick Freed Date: Fri, 16 Jun 2023 11:37:48 -0400 Subject: [PATCH 24/24] change new cmap-format tests to unified --- .../cmap-format/pool-checkout-error-auth.json | 68 ----- .../cmap-format/pool-checkout-error-auth.yml | 40 --- .../pool-checkout-error-clears-waitqueue.json | 105 ------- .../pool-checkout-error-clears-waitqueue.yml | 64 ----- .../pool-checkout-error-concurrent.json | 82 ------ .../pool-checkout-error-concurrent.yml | 47 --- .../pool-checkout-error-network.json | 67 ----- .../pool-checkout-error-network.yml | 39 --- .../checkout-error-clears-waitqueue.json | 269 ++++++++++++++++++ .../checkout-error-clears-waitqueue.yml | 162 +++++++++++ .../unified/concurrent-checkout-error.json | 250 ++++++++++++++++ .../unified/concurrent-checkout-error.yml | 149 ++++++++++ 12 files changed, 830 insertions(+), 512 deletions(-) delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json delete mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml create mode 100644 source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.json create mode 100644 source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.yml create mode 100644 source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.json create mode 100644 source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.yml diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json deleted file mode 100644 index 328da95c24..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "version": 1, - "style": "integration", - "description": "must properly handle auth error on check out", - "runOn": [ - { - "minServerVersion": "4.9.0", - "auth": true - } - ], - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 1 - }, - "data": { - "failCommands": [ - "saslContinue" - ], - "appName": "poolCheckOutAuthErrorTest", - "errorCode": 18 - } - }, - "poolOptions": { - "appName": "poolCheckOutAuthErrorTest" - }, - "operations": [ - { - "name": "ready" - }, - { - "name": "checkOut" - } - ], - "error": { - "type": "AuthenticationError" - }, - "events": [ - { - "type": "ConnectionCheckOutStarted", - "address": 42 - }, - { - "type": "ConnectionCreated", - "connectionId": 1, - "address": 42 - }, - { - "type": "ConnectionPoolCleared", - "address": 42 - }, - { - "type": "ConnectionClosed", - "connectionId": 1, - "address": 42, - "reason": "error" - }, - { - "type": "ConnectionCheckOutFailed", - "address": 42, - "reason": "connectionError" - } - ], - "ignore": [ - "ConnectionPoolCreated", - "ConnectionPoolReady" - ] -} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml deleted file mode 100644 index 9aecc785df..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-auth.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: 1 -style: integration -description: must properly handle auth error on check out -runOn: - - - # required for appName in fail point - minServerVersion: "4.9.0" - auth: true -failPoint: - configureFailPoint: failCommand - mode: { times: 1 } - data: - failCommands: ["saslContinue"] - appName: &appName "poolCheckOutAuthErrorTest" - errorCode: 18 -poolOptions: - appName: *appName -operations: - - name: ready - - name: checkOut -error: - type: AuthenticationError -events: - - type: ConnectionCheckOutStarted - address: 42 - - type: ConnectionCreated - connectionId: 1 - address: 42 - - type: ConnectionPoolCleared - address: 42 - - type: ConnectionClosed - connectionId: 1 - address: 42 - reason: error - - type: ConnectionCheckOutFailed - address: 42 - reason: connectionError -ignore: - - ConnectionPoolCreated - - ConnectionPoolReady diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json deleted file mode 100644 index 8fef127e7d..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "version": 1, - "style": "integration", - "description": "connection error during checkout clears wait queue", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 50 - }, - "data": { - "failCommands": [ - "hello", - "isMaster" - ], - "appName": "poolCheckOutErrorWaitQueueTest", - "closeConnection": true, - "blockConnection": true, - "blockTimeMS": 1000 - } - }, - "poolOptions": { - "maxPoolSize": 1, - "waitQueueTimeoutMS": 30000, - "appName": "poolCheckOutErrorWaitQueueTest" - }, - "operations": [ - { - "name": "ready" - }, - { - "name": "start", - "target": "thread1" - }, - { - "name": "checkOut", - "thread": "thread1" - }, - { - "name": "start", - "target": "thread2" - }, - { - "name": "checkOut", - "thread": "thread2" - }, - { - "name": "start", - "target": "thread3" - }, - { - "name": "checkOut", - "thread": "thread3" - }, - { - "name": "waitForEvent", - "event": "ConnectionCheckOutStarted", - "count": 3 - }, - { - "name": "waitForEvent", - "event": "ConnectionCheckOutFailed", - "count": 3, - "timeout": 2000 - } - ], - "events": [ - { - "type": "ConnectionCheckOutStarted", - "address": 42 - }, - { - "type": "ConnectionCheckOutStarted", - "address": 42 - }, - { - "type": "ConnectionCheckOutStarted", - "address": 42 - }, - { - "type": "ConnectionPoolCleared", - "address": 42 - }, - { - "type": "ConnectionCheckOutFailed", - "reason": "connectionError", - "address": 42 - }, - { - "type": "ConnectionCheckOutFailed", - "reason": "connectionError", - "address": 42 - }, - { - "type": "ConnectionCheckOutFailed", - "reason": "connectionError", - "address": 42 - } - ], - "ignore": [ - "ConnectionPoolReady", - "ConnectionCreated", - "ConnectionPoolCreated", - "ConnectionClosed" - ] -} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml deleted file mode 100644 index b2da320baa..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-clears-waitqueue.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: 1 -style: integration -description: connection error during checkout clears wait queue -failPoint: - configureFailPoint: failCommand - # high amount to ensure not interfered with by monitor checks. - mode: { times: 50 } - data: - failCommands: ["hello", "isMaster"] - appName: &appName "poolCheckOutErrorWaitQueueTest" - closeConnection: true - blockConnection: true - blockTimeMS: 1000 -poolOptions: - maxPoolSize: 1 - waitQueueTimeoutMS: 30000 - appName: *appName -operations: - - name: ready - # start a few threads that all will enter - # the wait queue - - name: start - target: thread1 - - name: checkOut - thread: thread1 - - name: start - target: thread2 - - name: checkOut - thread: thread2 - - name: start - target: thread3 - - name: checkOut - thread: thread3 - # wait for the 3 threads to start checking out a connection - - name: waitForEvent - event: ConnectionCheckOutStarted - count: 3 - - name: waitForEvent - event: ConnectionCheckOutFailed - count: 3 - timeout: 2000 -events: - - type: ConnectionCheckOutStarted - address: 42 - - type: ConnectionCheckOutStarted - address: 42 - - type: ConnectionCheckOutStarted - address: 42 - - type: ConnectionPoolCleared - address: 42 - - type: ConnectionCheckOutFailed - reason: connectionError - address: 42 - - type: ConnectionCheckOutFailed - reason: connectionError - address: 42 - - type: ConnectionCheckOutFailed - reason: connectionError - address: 42 -ignore: - - ConnectionPoolReady - - ConnectionCreated - - ConnectionPoolCreated - - ConnectionClosed diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json deleted file mode 100644 index c69c5580ee..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "version": 1, - "style": "integration", - "description": "must properly handle multiple concurrent network errors on check out", - "runOn": [ - { - "minServerVersion": "4.9.0" - } - ], - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 50 - }, - "data": { - "failCommands": [ - "isMaster", - "hello" - ], - "closeConnection": true, - "appName": "poolCheckOutErrorTest" - } - }, - "poolOptions": { - "appName": "poolCheckOutErrorTest" - }, - "operations": [ - { - "name": "ready" - }, - { - "name": "start", - "target": "thread1" - }, - { - "name": "start", - "target": "thread2" - }, - { - "name": "start", - "target": "thread3" - }, - { - "name": "checkOut", - "thread": "thread1" - }, - { - "name": "checkOut", - "thread": "thread2" - }, - { - "name": "checkOut", - "thread": "thread3" - }, - { - "name": "waitForEvent", - "event": "ConnectionCheckOutFailed", - "count": 3 - }, - { - "name": "ready" - } - ], - "events": [ - { - "type": "ConnectionPoolReady" - }, - { - "type": "ConnectionPoolCleared" - }, - { - "type": "ConnectionPoolReady" - } - ], - "ignore": [ - "ConnectionCheckOutStarted", - "ConnectionCheckOutFailed", - "ConnectionPoolCreated", - "ConnectionCreated", - "ConnectionClosed" - ] -} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml deleted file mode 100644 index 12dcfb2135..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-concurrent.yml +++ /dev/null @@ -1,47 +0,0 @@ -version: 1 -style: integration -description: must properly handle multiple concurrent network errors on check out -runOn: - - - # required for appName in fail point - minServerVersion: "4.9.0" -failPoint: - configureFailPoint: failCommand - # high amount to ensure not interfered with by monitor checks. - mode: { times: 50 } - data: - failCommands: ["isMaster","hello"] - closeConnection: true - appName: &appName "poolCheckOutErrorTest" -poolOptions: - appName: *appName -operations: - - name: ready - - name: start - target: thread1 - - name: start - target: thread2 - - name: start - target: thread3 - - name: checkOut - thread: thread1 - - name: checkOut - thread: thread2 - - name: checkOut - thread: thread3 - - name: waitForEvent - event: ConnectionCheckOutFailed - count: 3 - - name: ready -events: - - type: ConnectionPoolReady - # Pool should only be cleared once. - - type: ConnectionPoolCleared - - type: ConnectionPoolReady -ignore: - - ConnectionCheckOutStarted - - ConnectionCheckOutFailed - - ConnectionPoolCreated - - ConnectionCreated - - ConnectionClosed - diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json deleted file mode 100644 index ee04600d67..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "version": 1, - "style": "integration", - "description": "must properly handle network error on check out", - "runOn": [ - { - "minServerVersion": "4.9.0" - } - ], - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 50 - }, - "data": { - "failCommands": [ - "isMaster", - "hello" - ], - "closeConnection": true, - "appName": "poolCheckOutErrorTest" - } - }, - "poolOptions": { - "appName": "poolCheckOutErrorTest" - }, - "operations": [ - { - "name": "ready" - }, - { - "name": "checkOut" - } - ], - "error": { - "type": "NetworkError" - }, - "events": [ - { - "type": "ConnectionCheckOutStarted", - "address": 42 - }, - { - "type": "ConnectionCreated", - "address": 42 - }, - { - "type": "ConnectionPoolCleared", - "address": 42 - }, - { - "type": "ConnectionClosed", - "address": 42, - "connectionId": 1, - "reason": "error" - }, - { - "type": "ConnectionCheckOutFailed", - "address": 42, - "reason": "connectionError" - } - ], - "ignore": [ - "ConnectionPoolCreated", - "ConnectionPoolReady" - ] -} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml deleted file mode 100644 index 43c8f44bc0..0000000000 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-checkout-error-network.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: 1 -style: integration -description: must properly handle network error on check out -runOn: - - - # required for appName in fail point - minServerVersion: "4.9.0" -failPoint: - configureFailPoint: failCommand - # high amount to ensure not interfered with by monitor checks. - mode: { times: 50 } - data: - failCommands: ["isMaster","hello"] - closeConnection: true - appName: &appName "poolCheckOutErrorTest" -poolOptions: - appName: *appName -operations: - - name: ready - - name: checkOut -error: - type: NetworkError -events: - - type: ConnectionCheckOutStarted - address: 42 - - type: ConnectionCreated - address: 42 - - type: ConnectionPoolCleared - address: 42 - - type: ConnectionClosed - address: 42 - connectionId: 1 - reason: error - - type: ConnectionCheckOutFailed - address: 42 - reason: connectionError -ignore: - - ConnectionPoolCreated - - ConnectionPoolReady diff --git a/source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.json b/source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.json new file mode 100644 index 0000000000..7a87fc8e57 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.json @@ -0,0 +1,269 @@ +{ + "description": "checkout-error-clears-waitqueue", + "schemaVersion": "1.10", + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Connection error during checkout clears WaitQueue", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "connectionCreatedEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "retryReads": false, + "appname": "poolCheckOutErrorWaitQueueTest", + "serverSelectionTimeoutMS": 100, + "waitQueueTimeoutMS": 30000, + "directConnection": true, + "maxPoolSize": 1 + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "cmap-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "cmap-network-error" + } + }, + { + "thread": { + "id": "thread0" + } + }, + { + "thread": { + "id": "thread1" + } + }, + { + "thread": { + "id": "thread2" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolReadyEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 50 + }, + "data": { + "failCommands": [ + "hello", + "isMaster" + ], + "appName": "poolCheckOutErrorWaitQueueTest", + "closeConnection": true, + "blockConnection": true, + "blockTimeMS": 1000 + } + } + } + }, + { + "name": "runOnThread", + "object": "testRunner", + "arguments": { + "thread": "thread0", + "operation": { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + } + } + }, + { + "name": "runOnThread", + "object": "testRunner", + "arguments": { + "thread": "thread1", + "operation": { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + } + } + }, + { + "name": "runOnThread", + "object": "testRunner", + "arguments": { + "thread": "thread2", + "operation": { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + } + } + }, + { + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread0" + } + }, + { + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread1" + } + }, + { + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread2" + } + }, + { + "name": "assertEventCount", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionClosedEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "connectionCreatedEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.yml b/source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.yml new file mode 100644 index 0000000000..c070cef4f3 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/checkout-error-clears-waitqueue.yml @@ -0,0 +1,162 @@ +--- +description: checkout-error-clears-waitqueue + +schemaVersion: "1.10" + +createEntities: + - client: + id: &setupClient setupClient + useMultipleMongoses: false + +initialData: &initialData + - collectionName: &collectionName cmap-network-error + databaseName: &databaseName cmap-tests + documents: + - _id: 1 + - _id: 2 + +tests: + - description: Connection error during checkout clears WaitQueue + runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single, replicaset, sharded ] + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: + - connectionCreatedEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - poolClearedEvent + uriOptions: + retryWrites: false + retryReads: false + appname: &appName "poolCheckOutErrorWaitQueueTest" + serverSelectionTimeoutMS: 100 + waitQueueTimeoutMS: 30000 + directConnection: true + maxPoolSize: 1 + - database: + id: &database database + client: *client + databaseName: *databaseName + - collection: + id: &collection collection + database: *database + collectionName: *collectionName + - thread: + id: &thread0 thread0 + - thread: + id: &thread1 thread1 + - thread: + id: &thread2 thread2 + # wait for the server to be discovered before enabling the failpoint + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + poolReadyEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: *setupClient + failPoint: + configureFailPoint: failCommand + # high amount to ensure not interfered with by monitor checks. + mode: { times: 50 } + data: + failCommands: ["hello", "isMaster"] + appName: *appName + closeConnection: true + blockConnection: true + blockTimeMS: 1000 + # Start three concurrent checkOut attempts. + # Since maxPoolSize=1 and hello will block for a while, two of the threads will enter the WaitQueue. + # Once the checkOut finally fails, the pool will be cleared and the two other threads will be evicted from the + # WaitQueue without ever having attempted to create a connection. + - name: runOnThread + object: testRunner + arguments: + thread: *thread0 + operation: + name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: runOnThread + object: testRunner + arguments: + thread: *thread1 + operation: + name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: runOnThread + object: testRunner + arguments: + thread: *thread2 + operation: + name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: waitForThread + object: testRunner + arguments: + thread: *thread0 + - name: waitForThread + object: testRunner + arguments: + thread: *thread1 + - name: waitForThread + object: testRunner + arguments: + thread: *thread2 + - name: assertEventCount + object: testRunner + arguments: + client: *client + event: + connectionClosedEvent: {} + count: 1 + expectEvents: + - client: *client + eventType: cmap + events: + # Only one ConnectionCreatedEvent should be emitted, since the other threads should be evicted by + # the pool clear without ever attempting to create a connection. + - connectionCreatedEvent: {} + - poolClearedEvent: {} + - connectionCheckOutFailedEvent: + reason: connectionError + - connectionCheckOutFailedEvent: + reason: connectionError + - connectionCheckOutFailedEvent: + reason: connectionError + + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - _id: 1 + - _id: 2 diff --git a/source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.json b/source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.json new file mode 100644 index 0000000000..e9a39dbc44 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.json @@ -0,0 +1,250 @@ +{ + "description": "concurrent-checkout-error", + "schemaVersion": "1.10", + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Pool properly handles multiple concurrent network errors on check out", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "poolReadyEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "retryReads": false, + "appname": "poolConcurrentCheckOutErrorTest", + "serverSelectionTimeoutMS": 100, + "waitQueueTimeoutMS": 30000, + "directConnection": true + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "cmap-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "cmap-network-error" + } + }, + { + "thread": { + "id": "thread0" + } + }, + { + "thread": { + "id": "thread1" + } + }, + { + "thread": { + "id": "thread2" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolReadyEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 50 + }, + "data": { + "failCommands": [ + "hello", + "isMaster" + ], + "appName": "poolConcurrentCheckOutErrorTest", + "closeConnection": true, + "blockConnection": true, + "blockTimeMS": 1000 + } + } + } + }, + { + "name": "runOnThread", + "object": "testRunner", + "arguments": { + "thread": "thread0", + "operation": { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + } + } + }, + { + "name": "runOnThread", + "object": "testRunner", + "arguments": { + "thread": "thread1", + "operation": { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + } + } + }, + { + "name": "runOnThread", + "object": "testRunner", + "arguments": { + "thread": "thread2", + "operation": { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "isError": true + } + } + } + }, + { + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread0" + } + }, + { + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread1" + } + }, + { + "name": "waitForThread", + "object": "testRunner", + "arguments": { + "thread": "thread2" + } + }, + { + "name": "assertEventCount", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolClearedEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "poolReadyEvent": {} + }, + { + "poolClearedEvent": {} + } + ] + } + ], + "outcome": [ + { + "collectionName": "cmap-network-error", + "databaseName": "cmap-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.yml b/source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.yml new file mode 100644 index 0000000000..7f37627682 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/unified/concurrent-checkout-error.yml @@ -0,0 +1,149 @@ +--- +description: concurrent-checkout-error + +schemaVersion: "1.10" + +createEntities: + - client: + id: &setupClient setupClient + useMultipleMongoses: false + +initialData: &initialData + - collectionName: &collectionName cmap-network-error + databaseName: &databaseName cmap-tests + documents: + - _id: 1 + - _id: 2 + +tests: + - description: Pool properly handles multiple concurrent network errors on check out + runOnRequirements: + # failCommand appName requirements + - minServerVersion: "4.4" + serverless: forbid + topologies: [ single, replicaset, sharded ] + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: &client client + useMultipleMongoses: false + observeEvents: + - poolReadyEvent + - poolClearedEvent + uriOptions: + retryWrites: false + retryReads: false + appname: &appName "poolConcurrentCheckOutErrorTest" + serverSelectionTimeoutMS: 100 + waitQueueTimeoutMS: 30000 + directConnection: true + - database: + id: &database database + client: *client + databaseName: *databaseName + - collection: + id: &collection collection + database: *database + collectionName: *collectionName + - thread: + id: &thread0 thread0 + - thread: + id: &thread1 thread1 + - thread: + id: &thread2 thread2 + # wait for the server to be discovered before enabling the failpoint + - name: waitForEvent + object: testRunner + arguments: + client: *client + event: + poolReadyEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: *setupClient + failPoint: + configureFailPoint: failCommand + # high amount to ensure not interfered with by monitor checks. + mode: { times: 50 } + data: + failCommands: ["hello", "isMaster"] + appName: *appName + closeConnection: true + blockConnection: true + blockTimeMS: 1000 + # Start three concurrent checkOut attempts. The failpoint blocks for 1s before closing the connection, so all + # three checkOuts should receive network errors. + - name: runOnThread + object: testRunner + arguments: + thread: *thread0 + operation: + name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: runOnThread + object: testRunner + arguments: + thread: *thread1 + operation: + name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: runOnThread + object: testRunner + arguments: + thread: *thread2 + operation: + name: find + object: *collection + arguments: + filter: + _id: 1 + expectError: + isError: true + - name: waitForThread + object: testRunner + arguments: + thread: *thread0 + - name: waitForThread + object: testRunner + arguments: + thread: *thread1 + - name: waitForThread + object: testRunner + arguments: + thread: *thread2 + - name: assertEventCount + object: testRunner + arguments: + client: *client + event: + poolClearedEvent: {} + count: 1 + expectEvents: + - client: *client + eventType: cmap + events: + # The pool should only be cleared once despite the multiple errors. + - poolReadyEvent: {} + - poolClearedEvent: {} + + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - _id: 1 + - _id: 2