From a23518b12679650c7c0749a5bac7e96bb1e9c7fa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 13:21:31 -0600 Subject: [PATCH 01/12] PYTHON-5528 Add exponential backoff to operation retry loop for server overloaded errors (cherry picked from commit 95948d3def875f1efb4d912ab8f507f518604ef5) --- .evergreen/resync-specs.sh | 3 + test/asynchronous/test_client_backpressure.py | 59 + .../backpressure-retry-loop.json | 3376 ++++++++++++++++ .../backpressure-retry-max-attempts.json | 3413 +++++++++++++++++ tools/synchro.py | 1 + 5 files changed, 6852 insertions(+) create mode 100644 test/asynchronous/test_client_backpressure.py create mode 100644 test/client-backpressure/backpressure-retry-loop.json create mode 100644 test/client-backpressure/backpressure-retry-max-attempts.json diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index d2bd89c781..4bb9c86304 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -94,6 +94,9 @@ do change-streams|change_streams) cpjson change-streams/tests/ change_streams/ ;; + client-backpressure|client_backpressure) + cpjson client-backpressure/tests client-backpressure + ;; client-side-encryption|csfle|fle) cpjson client-side-encryption/tests/ client-side-encryption/spec cpjson client-side-encryption/corpus/ client-side-encryption/corpus diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py new file mode 100644 index 0000000000..669be87aca --- /dev/null +++ b/test/asynchronous/test_client_backpressure.py @@ -0,0 +1,59 @@ +# Copyright 2025-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os +import sys + +sys.path[0:0] = [""] + +from test.asynchronous import ( + AsyncIntegrationTest, + async_client_context, + unittest, +) +from test.asynchronous.unified_format import generate_test_classes +from test.utils_shared import EventListener, OvertCommandListener + +_IS_SYNC = False + + +class AsyncTestClientBackpressure(AsyncIntegrationTest): + listener: EventListener + + @classmethod + def setUpClass(cls) -> None: + cls.listener = OvertCommandListener() + + @async_client_context.require_connection + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.listener.reset() + self.client = await self.async_rs_or_single_client( + event_listeners=[self.listener], retryWrites=False + ) + + +_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") + + +globals().update( + generate_test_classes( + _TEST_PATH, + module=__name__, + ) +) + +if __name__ == "__main__": + unittest.main() diff --git a/test/client-backpressure/backpressure-retry-loop.json b/test/client-backpressure/backpressure-retry-loop.json new file mode 100644 index 0000000000..b121d9bd3d --- /dev/null +++ b/test/client-backpressure/backpressure-retry-loop.json @@ -0,0 +1,3376 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "utilDb", + "client": "failPointClient", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "utilCollection", + "database": "utilDb", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "object": "utilCollection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/client-backpressure/backpressure-retry-max-attempts.json b/test/client-backpressure/backpressure-retry-max-attempts.json new file mode 100644 index 0000000000..a499aa490b --- /dev/null +++ b/test/client-backpressure/backpressure-retry-max-attempts.json @@ -0,0 +1,3413 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/tools/synchro.py b/tools/synchro.py index 661d8988cb..c87b4e8139 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -212,6 +212,7 @@ def async_only_test(f: str) -> bool: "test_backpressure.py", "test_change_stream.py", "test_client.py", + "test_client_backpressure.py", "test_client_bulk_write.py", "test_client_context.py", "test_client_metadata.py", From 9c6eeed5f768b4ad3ffe6cb605d8456c8a3c0ea1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 13:22:10 -0600 Subject: [PATCH 02/12] add async test (cherry picked from commit ebf65519f12d160da6d2088c22f6c69fad5a37d0) --- test/test_client_backpressure.py | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 test/test_client_backpressure.py diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py new file mode 100644 index 0000000000..5492f5f271 --- /dev/null +++ b/test/test_client_backpressure.py @@ -0,0 +1,57 @@ +# Copyright 2025-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os +import sys + +sys.path[0:0] = [""] + +from test import ( + IntegrationTest, + client_context, + unittest, +) +from test.unified_format import generate_test_classes +from test.utils_shared import EventListener, OvertCommandListener + +_IS_SYNC = True + + +class TestClientBackpressure(IntegrationTest): + listener: EventListener + + @classmethod + def setUpClass(cls) -> None: + cls.listener = OvertCommandListener() + + @client_context.require_connection + def setUp(self) -> None: + super().setUp() + self.listener.reset() + self.client = self.rs_or_single_client(event_listeners=[self.listener], retryWrites=False) + + +_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") + + +globals().update( + generate_test_classes( + _TEST_PATH, + module=__name__, + ) +) + +if __name__ == "__main__": + unittest.main() From f83e16bc31f71fc15d75ac60f91a96c08cf09650 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 14:05:58 -0600 Subject: [PATCH 03/12] fix test setup --- test/asynchronous/test_client_backpressure.py | 8 ++++++-- test/test_client_backpressure.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index 669be87aca..183b03fdb1 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import pathlib import sys sys.path[0:0] = [""] @@ -45,8 +46,11 @@ async def asyncSetUp(self) -> None: ) -_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") - +# Location of JSON test specifications. +if _IS_SYNC: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent, "client-backpressure") +else: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent.parent, "client-backpressure") globals().update( generate_test_classes( diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index 5492f5f271..2dd9187ebb 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import pathlib import sys sys.path[0:0] = [""] @@ -43,8 +44,11 @@ def setUp(self) -> None: self.client = self.rs_or_single_client(event_listeners=[self.listener], retryWrites=False) -_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "client-backpressure") - +# Location of JSON test specifications. +if _IS_SYNC: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent, "client-backpressure") +else: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent.parent, "client-backpressure") globals().update( generate_test_classes( From 6333244098ccedd3cf7ae272d80c20195e2df0da Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 14:59:34 -0600 Subject: [PATCH 04/12] implement handshake prose test --- pymongo/asynchronous/pool.py | 1 + pymongo/synchronous/pool.py | 1 + test/asynchronous/test_client_metadata.py | 13 +++++++++++++ test/test_client_metadata.py | 13 +++++++++++++ 4 files changed, 28 insertions(+) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 2c9c70cd46..ea41da27ec 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -269,6 +269,7 @@ async def _hello( cmd = self.hello_cmd() performing_handshake = not self.performed_handshake awaitable = False + cmd["backpressure"] = True if performing_handshake: self.performed_handshake = True cmd["client"] = self.opts.metadata diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 9e50fa5903..c833cd8a46 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -269,6 +269,7 @@ def _hello( cmd = self.hello_cmd() performing_handshake = not self.performed_handshake awaitable = False + cmd["backpressure"] = True if performing_handshake: self.performed_handshake = True cmd["client"] = self.opts.metadata diff --git a/test/asynchronous/test_client_metadata.py b/test/asynchronous/test_client_metadata.py index 2f175cceed..a0fdb1c269 100644 --- a/test/asynchronous/test_client_metadata.py +++ b/test/asynchronous/test_client_metadata.py @@ -227,6 +227,19 @@ async def test_duplicate_driver_name_no_op(self): # add same metadata again await self.check_metadata_added(client, "Framework", None, None) + async def test_handshake_documents_include_backpressure(self): + # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of + # connection establishment. + client = await self.async_rs_or_single_client("mongodb://" + self.server.address_string) + + # Send a `ping` command to the server and verify that the command succeeds. This ensure that a connection is + # established on all topologies. Note: MockupDB only supports standalone servers. + await client.admin.command("ping") + + # Assert that for every handshake document intercepted: + # the document has a field `backpressure` whose value is `true`. + self.assertEqual(self.handshake_req["backpressure"], True) + if __name__ == "__main__": unittest.main() diff --git a/test/test_client_metadata.py b/test/test_client_metadata.py index a94c5aa25e..8132378936 100644 --- a/test/test_client_metadata.py +++ b/test/test_client_metadata.py @@ -227,6 +227,19 @@ def test_duplicate_driver_name_no_op(self): # add same metadata again self.check_metadata_added(client, "Framework", None, None) + def test_handshake_documents_include_backpressure(self): + # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of + # connection establishment. + client = self.rs_or_single_client("mongodb://" + self.server.address_string) + + # Send a `ping` command to the server and verify that the command succeeds. This ensure that a connection is + # established on all topologies. Note: MockupDB only supports standalone servers. + client.admin.command("ping") + + # Assert that for every handshake document intercepted: + # the document has a field `backpressure` whose value is `true`. + self.assertEqual(self.handshake_req["backpressure"], True) + if __name__ == "__main__": unittest.main() From 46440748e4b0ccb78303dc11bad9815ac3a19915 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Dec 2025 16:55:24 -0600 Subject: [PATCH 05/12] update unified test and add prose test --- test/asynchronous/test_client_backpressure.py | 53 ++- .../backpressure-retry-loop.json | 384 ------------------ test/test_client_backpressure.py | 55 ++- 3 files changed, 106 insertions(+), 386 deletions(-) diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index 183b03fdb1..db76e797e2 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -16,6 +16,10 @@ import os import pathlib import sys +from time import perf_counter +from unittest.mock import patch + +from pymongo.errors import OperationFailure sys.path[0:0] = [""] @@ -41,10 +45,57 @@ def setUpClass(cls) -> None: async def asyncSetUp(self) -> None: await super().asyncSetUp() self.listener.reset() + self.app_name = self.__class__.__name__.lower() self.client = await self.async_rs_or_single_client( - event_listeners=[self.listener], retryWrites=False + event_listeners=[self.listener], retryWrites=False, appName=self.app_name ) + @patch("random.random") + async def test_01_operation_retry_uses_exponential_backoff(self, random_func): + # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + + # 1. let `client` be a `MongoClient` + client = self.client + + # 2. let `collection` be a collection + collection = client.test.test + + # 3. Now, run transactions without backoff: + + # a. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + random_func.return_value = 0 + + # b. Configure the following failPoint: + fail_point = dict( + mode="alwaysOn", + data=dict( + failCommands=["insert"], + errorCode=2, + errorLabels=["SystemOverloadedError", "RetryableError"], + appName=self.app_name, + ), + ) + async with self.fail_point(fail_point): + # c. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + start0 = perf_counter() + with self.assertRaises(OperationFailure): + await collection.insert_one({"a": 1}) + end0 = perf_counter() + + # d. Configure the random number generator used for jitter to always return `1`. + random_func.return_value = 1 + + # e. Execute step c again. + start1 = perf_counter() + with self.assertRaises(OperationFailure): + await collection.insert_one({"a": 1}) + end1 = perf_counter() + + # f. Compare the two time between the two runs. + # The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + # runs. + self.assertTrue(abs((end1 - start1) - (end0 - start0 + 3.1)) < 1) + # Location of JSON test specifications. if _IS_SYNC: diff --git a/test/client-backpressure/backpressure-retry-loop.json b/test/client-backpressure/backpressure-retry-loop.json index b121d9bd3d..20bdfe3a69 100644 --- a/test/client-backpressure/backpressure-retry-loop.json +++ b/test/client-backpressure/backpressure-retry-loop.json @@ -85,18 +85,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -186,18 +174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -284,18 +260,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -391,18 +355,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -502,18 +454,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -610,18 +550,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -711,18 +639,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -812,18 +728,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -916,18 +820,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1018,18 +910,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1119,18 +999,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1220,18 +1088,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1318,18 +1174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1420,18 +1264,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1521,18 +1353,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1622,18 +1442,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1720,18 +1528,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1818,18 +1614,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1920,18 +1704,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2024,18 +1796,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2130,18 +1890,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2231,18 +1979,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2332,18 +2068,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2436,18 +2160,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2542,18 +2254,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2648,18 +2348,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2749,18 +2437,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2853,18 +2529,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2959,18 +2623,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3069,18 +2721,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3173,18 +2813,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "object": "utilCollection", "name": "createIndex", @@ -3284,18 +2912,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index 2dd9187ebb..55f34a77fb 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -16,6 +16,10 @@ import os import pathlib import sys +from time import perf_counter +from unittest.mock import patch + +from pymongo.errors import OperationFailure sys.path[0:0] = [""] @@ -41,7 +45,56 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() self.listener.reset() - self.client = self.rs_or_single_client(event_listeners=[self.listener], retryWrites=False) + self.app_name = self.__class__.__name__.lower() + self.client = self.rs_or_single_client( + event_listeners=[self.listener], retryWrites=False, appName=self.app_name + ) + + @patch("random.random") + def test_01_operation_retry_uses_exponential_backoff(self, random_func): + # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + + # 1. let `client` be a `MongoClient` + client = self.client + + # 2. let `collection` be a collection + collection = client.test.test + + # 3. Now, run transactions without backoff: + + # a. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + random_func.return_value = 0 + + # b. Configure the following failPoint: + fail_point = dict( + mode="alwaysOn", + data=dict( + failCommands=["insert"], + errorCode=2, + errorLabels=["SystemOverloadedError", "RetryableError"], + appName=self.app_name, + ), + ) + with self.fail_point(fail_point): + # c. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + start0 = perf_counter() + with self.assertRaises(OperationFailure): + collection.insert_one({"a": 1}) + end0 = perf_counter() + + # d. Configure the random number generator used for jitter to always return `1`. + random_func.return_value = 1 + + # e. Execute step c again. + start1 = perf_counter() + with self.assertRaises(OperationFailure): + collection.insert_one({"a": 1}) + end1 = perf_counter() + + # f. Compare the two time between the two runs. + # The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + # runs. + self.assertTrue(abs((end1 - start1) - (end0 - start0 + 3.1)) < 1) # Location of JSON test specifications. From 4ba159c2c8b2a2ae63685ce21de3d37d45c58072 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 06:16:01 -0600 Subject: [PATCH 06/12] fix overload retry for clientBulk --- pymongo/asynchronous/client_bulk.py | 14 +++++++++++++- pymongo/synchronous/client_bulk.py | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 151942c8a8..508b8e41c3 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -563,9 +563,21 @@ async def _execute_command( error, ConnectionFailure ) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError)) + retryable_label_error = ( + hasattr(error, "details") + and isinstance(error.details, dict) + and "errorLabels" in error.details + and isinstance(error.details["errorLabels"], list) + and "RetryableError" in error.details["errorLabels"] + ) + # Synthesize the full bulk result without modifying the # current one because this write operation may be retried. - if retryable and (retryable_top_level_error or retryable_network_error): + if retryable and ( + retryable_top_level_error + or retryable_network_error + or retryable_label_error + ): full = copy.deepcopy(full_result) _merge_command(self.ops, self.idx_offset, full, result) _throw_client_bulk_write_exception(full, self.verbose_results) diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index a606d028e1..e8167bcedc 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -561,9 +561,21 @@ def _execute_command( error, ConnectionFailure ) and not isinstance(error, (NotPrimaryError, WaitQueueTimeoutError)) + retryable_label_error = ( + hasattr(error, "details") + and isinstance(error.details, dict) + and "errorLabels" in error.details + and isinstance(error.details["errorLabels"], list) + and "RetryableError" in error.details["errorLabels"] + ) + # Synthesize the full bulk result without modifying the # current one because this write operation may be retried. - if retryable and (retryable_top_level_error or retryable_network_error): + if retryable and ( + retryable_top_level_error + or retryable_network_error + or retryable_label_error + ): full = copy.deepcopy(full_result) _merge_command(self.ops, self.idx_offset, full, result) _throw_client_bulk_write_exception(full, self.verbose_results) From 13338814a3a403579f1d7bb482981efa002417dc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 09:14:01 -0600 Subject: [PATCH 07/12] fix prose test --- test/asynchronous/test_client_backpressure.py | 1 + test/test_client_backpressure.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/test_client_backpressure.py b/test/asynchronous/test_client_backpressure.py index db76e797e2..b66497d9ee 100644 --- a/test/asynchronous/test_client_backpressure.py +++ b/test/asynchronous/test_client_backpressure.py @@ -51,6 +51,7 @@ async def asyncSetUp(self) -> None: ) @patch("random.random") + @async_client_context.require_failCommand_appName async def test_01_operation_retry_uses_exponential_backoff(self, random_func): # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. diff --git a/test/test_client_backpressure.py b/test/test_client_backpressure.py index 55f34a77fb..b6776d53ba 100644 --- a/test/test_client_backpressure.py +++ b/test/test_client_backpressure.py @@ -51,6 +51,7 @@ def setUp(self) -> None: ) @patch("random.random") + @client_context.require_failCommand_appName def test_01_operation_retry_uses_exponential_backoff(self, random_func): # Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. From 9a88851f34573bc1ed8c38613532eebb7ff7d774 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Thu, 4 Dec 2025 11:58:10 -0500 Subject: [PATCH 08/12] PYTHON-5664 extract using `tar` command (#2636) (cherry picked from commit 3093a7c7cbd048a84a7014c03a6e295955397eb0) --- .evergreen/scripts/setup_tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 7908836f50..939423ffcc 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -1,12 +1,10 @@ from __future__ import annotations import base64 -import io import os import platform import shutil import stat -import tarfile from pathlib import Path from urllib import request @@ -117,9 +115,10 @@ def setup_libmongocrypt(): LOGGER.info(f"Fetching {url}...") with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310 if response.status == 200: - fileobj = io.BytesIO(response.read()) - with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid: - fid.extractall(Path.cwd() / "libmongocrypt") + with Path("libmongocrypt.tar.gz").open("wb") as f: + f.write(response.read()) + Path("libmongocrypt").mkdir() + run_command("tar -xzf libmongocrypt.tar.gz -C libmongocrypt") LOGGER.info(f"Fetching {url}... done.") run_command("ls -la libmongocrypt") From 9e85611b8b84227de8134c382c8657b02390fc5e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 12:21:48 -0600 Subject: [PATCH 09/12] update justfile from master --- justfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/justfile b/justfile index 7ac5bd33ff..92bdee5be3 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,5 @@ # See https://just.systems/man/en/ for instructions set shell := ["bash", "-c"] -# Do not modify the lock file when running justfile commands. -export UV_FROZEN := "1" # Commonly used command segments. typing_run := "uv run --group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" @@ -16,7 +14,7 @@ default: [private] resync: - @uv sync --quiet --frozen + @uv sync --quiet install: bash .evergreen/scripts/setup-dev-env.sh @@ -50,12 +48,12 @@ typing-pyright: && resync {{typing_run}} pyright -p strict_pyrightconfig.json test/test_typing_strict.py [group('lint')] -lint: && resync - uv run pre-commit run --all-files +lint *args="": && resync + uvx pre-commit run --all-files {{args}} [group('lint')] -lint-manual: && resync - uv run pre-commit run --all-files --hook-stage manual +lint-manual *args="": && resync + uvx pre-commit run --all-files --hook-stage manual {{args}} [group('test')] test *args="-v --durations=5 --maxfail=10": && resync @@ -73,6 +71,10 @@ setup-tests *args="": teardown-tests: bash .evergreen/scripts/teardown-tests.sh +[group('test')] +integration-tests: + bash integration_tests/run.sh + [group('server')] run-server *args="": bash .evergreen/scripts/run-server.sh {{args}} From 77dc3b711e696627b73f25ff22e47498f697c0c7 Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Mon, 1 Dec 2025 15:15:09 -0500 Subject: [PATCH 10/12] PYTHON-5656: Fixes broken link to aggregation pipeline docs. (#2627) (cherry picked from commit 8bf826339110605c27e5f7338c075b3fdb26949e) --- pymongo/asynchronous/database.py | 2 +- pymongo/synchronous/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 9b84869315..5e34e52e69 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -701,7 +701,7 @@ async def aggregate( .. versionadded:: 3.9 .. _aggregation pipeline: - https://mongodb.com/docs/manual/reference/operator/aggregation-pipeline + https://www.mongodb.com/docs/manual/core/aggregation-pipeline/ .. _aggregate command: https://mongodb.com/docs/manual/reference/command/aggregate diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 6877854f46..27cc102d05 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -701,7 +701,7 @@ def aggregate( .. versionadded:: 3.9 .. _aggregation pipeline: - https://mongodb.com/docs/manual/reference/operator/aggregation-pipeline + https://www.mongodb.com/docs/manual/core/aggregation-pipeline/ .. _aggregate command: https://mongodb.com/docs/manual/reference/command/aggregate From 5dd20a1bcac1b55dabecd907e6819009e0ebcdef Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 13:53:10 -0600 Subject: [PATCH 11/12] Deprioritize previous servers for overload retry attempts --- pymongo/asynchronous/collection.py | 86 ++++++++++++++++++---------- pymongo/asynchronous/database.py | 54 +++++++++-------- pymongo/asynchronous/helpers.py | 29 ---------- pymongo/asynchronous/mongo_client.py | 18 +++--- pymongo/synchronous/collection.py | 82 +++++++++++++++++--------- pymongo/synchronous/database.py | 50 ++++++++++------ pymongo/synchronous/helpers.py | 29 ---------- pymongo/synchronous/mongo_client.py | 18 +++--- 8 files changed, 188 insertions(+), 178 deletions(-) diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index c6cb69af25..60755dbe89 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -20,7 +20,6 @@ from typing import ( TYPE_CHECKING, Any, - AsyncContextManager, Callable, Coroutine, Generic, @@ -58,7 +57,6 @@ AsyncCursor, AsyncRawBatchCursor, ) -from pymongo.asynchronous.helpers import _retry_overload from pymongo.collation import validate_collation_or_none from pymongo.common import _ecoc_coll_name, _esc_coll_name from pymongo.errors import ( @@ -573,11 +571,6 @@ async def watch( await change_stream._initialize_cursor() return change_stream - async def _conn_for_writes( - self, session: Optional[AsyncClientSession], operation: str - ) -> AsyncContextManager[AsyncConnection]: - return await self._database.client._conn_for_writes(session, operation) - async def _command( self, conn: AsyncConnection, @@ -654,7 +647,10 @@ async def _create_helper( if "size" in options: options["size"] = float(options["size"]) cmd.update(options) - async with await self._conn_for_writes(session, operation=_Op.CREATE) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: if qev2_required and conn.max_wire_version < 21: raise ConfigurationError( "Driver support of Queryable Encryption is incompatible with server. " @@ -671,6 +667,8 @@ async def _create_helper( session=session, ) + await self.database.client._retryable_write(False, inner, session, _Op.CREATE) + async def _create( self, options: MutableMapping[str, Any], @@ -2229,7 +2227,6 @@ async def create_indexes( return await self._create_indexes(indexes, session, **kwargs) @_csot.apply - @_retry_overload async def _create_indexes( self, indexes: Sequence[IndexModel], session: Optional[AsyncClientSession], **kwargs: Any ) -> list[str]: @@ -2243,7 +2240,10 @@ async def _create_indexes( command (like maxTimeMS) can be passed as keyword arguments. """ names = [] - async with await self._conn_for_writes(session, operation=_Op.CREATE_INDEXES) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> list[str]: supports_quorum = conn.max_wire_version >= 9 def gen_indexes() -> Iterator[Mapping[str, Any]]: @@ -2272,7 +2272,11 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: write_concern=self._write_concern_for(session), session=session, ) - return names + return names + + return await self.database.client._retryable_write( + False, inner, session, _Op.CREATE_INDEXES + ) async def create_index( self, @@ -2474,7 +2478,6 @@ async def drop_index( await self._drop_index(index_or_name, session, comment, **kwargs) @_csot.apply - @_retry_overload async def _drop_index( self, index_or_name: _IndexKeyHint, @@ -2493,7 +2496,10 @@ async def _drop_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - async with await self._conn_for_writes(session, operation=_Op.DROP_INDEXES) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: await self._command( conn, cmd, @@ -2503,6 +2509,8 @@ async def _drop_index( session=session, ) + await self.database.client._retryable_write(False, inner, session, _Op.DROP_INDEXES) + async def list_indexes( self, session: Optional[AsyncClientSession] = None, @@ -2766,17 +2774,22 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: cmd = {"createSearchIndexes": self.name, "indexes": list(gen_indexes())} cmd.update(kwargs) - async with await self._conn_for_writes( - session, operation=_Op.CREATE_SEARCH_INDEXES - ) as conn: + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> list[str]: resp = await self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) return [index["name"] for index in resp["indexesCreated"]] + return self.database.client._retryable_write( + False, inner, session, _Op.CREATE_SEARCH_INDEXES + ) + async def drop_search_index( self, name: str, @@ -2802,15 +2815,21 @@ async def drop_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - async with await self._conn_for_writes(session, operation=_Op.DROP_SEARCH_INDEXES) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: await self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + async def update_search_index( self, name: str, @@ -2838,15 +2857,21 @@ async def update_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - async with await self._conn_for_writes(session, operation=_Op.UPDATE_SEARCH_INDEX) as conn: + + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> None: await self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + async def options( self, session: Optional[AsyncClientSession] = None, @@ -3075,7 +3100,6 @@ async def aggregate_raw_batches( ) @_csot.apply - @_retry_overload async def rename( self, new_name: str, @@ -3127,17 +3151,21 @@ async def rename( if comment is not None: cmd["comment"] = comment write_concern = self._write_concern_for_cmd(cmd, session) + client = self._database.client - async with await self._conn_for_writes(session, operation=_Op.RENAME) as conn: - async with self._database.client._tmp_session(session) as s: - return await conn.command( - "admin", - cmd, - write_concern=write_concern, - parse_write_concern_error=True, - session=s, - client=self._database.client, - ) + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> MutableMapping[str, Any]: + return await conn.command( + "admin", + cmd, + write_concern=write_concern, + parse_write_concern_error=True, + session=session, + client=client, + ) + + return client._retryable_write(False, inner, session, _Op.RENAME) async def distinct( self, diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 5e34e52e69..7ad9017f70 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -38,7 +38,6 @@ from pymongo.asynchronous.change_stream import AsyncDatabaseChangeStream from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.command_cursor import AsyncCommandCursor -from pymongo.asynchronous.helpers import _retry_overload from pymongo.common import _ecoc_coll_name, _esc_coll_name from pymongo.database_shared import _check_name, _CodecDocumentType from pymongo.errors import CollectionInvalid, InvalidOperation @@ -479,7 +478,6 @@ async def watch( return change_stream @_csot.apply - @_retry_overload async def create_collection( self, name: str, @@ -822,7 +820,6 @@ async def command( ... @_csot.apply - @_retry_overload async def command( self, command: Union[str, MutableMapping[str, Any]], @@ -935,14 +932,15 @@ async def command( if read_preference is None: read_preference = (session and session._txn_read_preference()) or ReadPreference.PRIMARY - async with await self._client._conn_for_reads( - read_preference, session, operation=command_name - ) as ( - connection, - read_preference, - ): + + async def inner( + session: Optional[AsyncClientSession], + _server: Server, + conn: AsyncConnection, + read_preference: _ServerMode, + ) -> Union[dict[str, Any], _CodecDocumentType]: return await self._command( - connection, + conn, command, value, check, @@ -953,8 +951,11 @@ async def command( **kwargs, ) + return await self._client._retryable_read( + inner, read_preference, session, command_name, None, False + ) + @_csot.apply - @_retry_overload async def cursor_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1021,17 +1022,17 @@ async def cursor_command( async with self._client._tmp_session(session) as tmp_session: opts = codec_options or DEFAULT_CODEC_OPTIONS - if read_preference is None: read_preference = ( tmp_session and tmp_session._txn_read_preference() ) or ReadPreference.PRIMARY - async with await self._client._conn_for_reads( - read_preference, tmp_session, command_name - ) as ( - conn, - read_preference, - ): + + async def inner( + session: Optional[AsyncClientSession], + _server: Server, + conn: AsyncConnection, + read_preference: _ServerMode, + ) -> AsyncCommandCursor[_DocumentType]: response = await self._command( conn, command, @@ -1040,7 +1041,7 @@ async def cursor_command( None, read_preference, opts, - session=tmp_session, + session=session, **kwargs, ) coll = self.get_collection("$cmd", read_preference=read_preference) @@ -1050,7 +1051,7 @@ async def cursor_command( response["cursor"], conn.address, max_await_time_ms=max_await_time_ms, - session=tmp_session, + session=session, comment=comment, ) await cmd_cursor._maybe_pin_connection(conn) @@ -1058,6 +1059,10 @@ async def cursor_command( else: raise InvalidOperation("Command does not return a cursor.") + return await self.client._retryable_read( + inner, read_preference, tmp_session, command_name, False + ) + async def _retryable_read_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1259,9 +1264,11 @@ async def _drop_helper( if comment is not None: command["comment"] = comment - async with await self._client._conn_for_writes(session, operation=_Op.DROP) as connection: + async def inner( + session: Optional[AsyncClientSession], conn: AsyncConnection, _retryable_write: bool + ) -> dict[str, Any]: return await self._command( - connection, + conn, command, allowable_errors=["ns not found", 26], write_concern=self._write_concern_for(session), @@ -1269,8 +1276,9 @@ async def _drop_helper( session=session, ) + return await self.client._retryable_write(False, inner, session, _Op.DROP) + @_csot.apply - @_retry_overload async def drop_collection( self, name_or_collection: Union[str, AsyncCollection[_DocumentTypeArg]], diff --git a/pymongo/asynchronous/helpers.py b/pymongo/asynchronous/helpers.py index 9dc87ad285..c008f64a63 100644 --- a/pymongo/asynchronous/helpers.py +++ b/pymongo/asynchronous/helpers.py @@ -30,7 +30,6 @@ from pymongo import _csot from pymongo.errors import ( OperationFailure, - PyMongoError, ) from pymongo.helpers_shared import _REAUTHENTICATION_REQUIRED_CODE from pymongo.lock import _async_create_lock @@ -164,34 +163,6 @@ async def should_retry(self, attempt: int, delay: float) -> bool: return True -def _retry_overload(func: F) -> F: - @functools.wraps(func) - async def inner(self: Any, *args: Any, **kwargs: Any) -> Any: - retry_policy = self._retry_policy - attempt = 0 - while True: - try: - res = await func(self, *args, **kwargs) - await retry_policy.record_success(retry=attempt > 0) - return res - except PyMongoError as exc: - if not exc.has_error_label("RetryableError"): - raise - attempt += 1 - delay = 0 - if exc.has_error_label("SystemOverloadedError"): - delay = retry_policy.backoff(attempt) - if not await retry_policy.should_retry(attempt, delay): - raise - - # Implement exponential backoff on retry. - if delay: - await asyncio.sleep(delay) - continue - - return cast(F, inner) - - async def _getaddrinfo( host: Any, port: Any, **kwargs: Any ) -> list[ diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 486e00ae46..7e9b5c9a9d 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -69,7 +69,6 @@ from pymongo.asynchronous.client_session import _EmptyServerSession from pymongo.asynchronous.command_cursor import AsyncCommandCursor from pymongo.asynchronous.helpers import ( - _retry_overload, _RetryPolicy, _TokenBucket, ) @@ -2403,7 +2402,6 @@ async def list_database_names( return [doc["name"] async for doc in res] @_csot.apply - @_retry_overload async def drop_database( self, name_or_database: Union[str, database.AsyncDatabase[_DocumentTypeArg]], @@ -2446,15 +2444,13 @@ async def drop_database( f"name_or_database must be an instance of str or a AsyncDatabase, not {type(name)}" ) - async with await self._conn_for_writes(session, operation=_Op.DROP_DATABASE) as conn: - await self[name]._command( - conn, - {"dropDatabase": 1, "comment": comment}, - read_preference=ReadPreference.PRIMARY, - write_concern=self._write_concern_for(session), - parse_write_concern_error=True, - session=session, - ) + return await self[name].command( + {"dropDatabase": 1, "comment": comment}, + read_preference=ReadPreference.PRIMARY, + write_concern=self._write_concern_for(session), + parse_write_concern_error=True, + session=session, + ) @_csot.apply async def bulk_write( diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 9ee8e63947..6444e75ad8 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -21,7 +21,6 @@ TYPE_CHECKING, Any, Callable, - ContextManager, Generic, Iterable, Iterator, @@ -89,7 +88,6 @@ Cursor, RawBatchCursor, ) -from pymongo.synchronous.helpers import _retry_overload from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline from pymongo.write_concern import DEFAULT_WRITE_CONCERN, WriteConcern, validate_boolean @@ -574,11 +572,6 @@ def watch( change_stream._initialize_cursor() return change_stream - def _conn_for_writes( - self, session: Optional[ClientSession], operation: str - ) -> ContextManager[Connection]: - return self._database.client._conn_for_writes(session, operation) - def _command( self, conn: Connection, @@ -655,7 +648,10 @@ def _create_helper( if "size" in options: options["size"] = float(options["size"]) cmd.update(options) - with self._conn_for_writes(session, operation=_Op.CREATE) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: if qev2_required and conn.max_wire_version < 21: raise ConfigurationError( "Driver support of Queryable Encryption is incompatible with server. " @@ -672,6 +668,8 @@ def _create_helper( session=session, ) + self.database.client._retryable_write(False, inner, session, _Op.CREATE) + def _create( self, options: MutableMapping[str, Any], @@ -2226,7 +2224,6 @@ def create_indexes( return self._create_indexes(indexes, session, **kwargs) @_csot.apply - @_retry_overload def _create_indexes( self, indexes: Sequence[IndexModel], session: Optional[ClientSession], **kwargs: Any ) -> list[str]: @@ -2240,7 +2237,10 @@ def _create_indexes( command (like maxTimeMS) can be passed as keyword arguments. """ names = [] - with self._conn_for_writes(session, operation=_Op.CREATE_INDEXES) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> list[str]: supports_quorum = conn.max_wire_version >= 9 def gen_indexes() -> Iterator[Mapping[str, Any]]: @@ -2269,7 +2269,9 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: write_concern=self._write_concern_for(session), session=session, ) - return names + return names + + return self.database.client._retryable_write(False, inner, session, _Op.CREATE_INDEXES) def create_index( self, @@ -2471,7 +2473,6 @@ def drop_index( self._drop_index(index_or_name, session, comment, **kwargs) @_csot.apply - @_retry_overload def _drop_index( self, index_or_name: _IndexKeyHint, @@ -2490,7 +2491,10 @@ def _drop_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - with self._conn_for_writes(session, operation=_Op.DROP_INDEXES) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: self._command( conn, cmd, @@ -2500,6 +2504,8 @@ def _drop_index( session=session, ) + self.database.client._retryable_write(False, inner, session, _Op.DROP_INDEXES) + def list_indexes( self, session: Optional[ClientSession] = None, @@ -2763,15 +2769,22 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: cmd = {"createSearchIndexes": self.name, "indexes": list(gen_indexes())} cmd.update(kwargs) - with self._conn_for_writes(session, operation=_Op.CREATE_SEARCH_INDEXES) as conn: + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> list[str]: resp = self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) return [index["name"] for index in resp["indexesCreated"]] + return self.database.client._retryable_write( + False, inner, session, _Op.CREATE_SEARCH_INDEXES + ) + def drop_search_index( self, name: str, @@ -2797,15 +2810,21 @@ def drop_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - with self._conn_for_writes(session, operation=_Op.DROP_SEARCH_INDEXES) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + def update_search_index( self, name: str, @@ -2833,15 +2852,21 @@ def update_search_index( cmd.update(kwargs) if comment is not None: cmd["comment"] = comment - with self._conn_for_writes(session, operation=_Op.UPDATE_SEARCH_INDEX) as conn: + + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> None: self._command( conn, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found", 26], codec_options=_UNICODE_REPLACE_CODEC_OPTIONS, + session=session, ) + return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + def options( self, session: Optional[ClientSession] = None, @@ -3068,7 +3093,6 @@ def aggregate_raw_batches( ) @_csot.apply - @_retry_overload def rename( self, new_name: str, @@ -3120,17 +3144,21 @@ def rename( if comment is not None: cmd["comment"] = comment write_concern = self._write_concern_for_cmd(cmd, session) + client = self._database.client - with self._conn_for_writes(session, operation=_Op.RENAME) as conn: - with self._database.client._tmp_session(session) as s: - return conn.command( - "admin", - cmd, - write_concern=write_concern, - parse_write_concern_error=True, - session=s, - client=self._database.client, - ) + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> MutableMapping[str, Any]: + return conn.command( + "admin", + cmd, + write_concern=write_concern, + parse_write_concern_error=True, + session=session, + client=client, + ) + + return client._retryable_write(False, inner, session, _Op.RENAME) def distinct( self, diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 27cc102d05..4fa8f4f1b4 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -43,7 +43,6 @@ from pymongo.synchronous.change_stream import DatabaseChangeStream from pymongo.synchronous.collection import Collection from pymongo.synchronous.command_cursor import CommandCursor -from pymongo.synchronous.helpers import _retry_overload from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline if TYPE_CHECKING: @@ -479,7 +478,6 @@ def watch( return change_stream @_csot.apply - @_retry_overload def create_collection( self, name: str, @@ -822,7 +820,6 @@ def command( ... @_csot.apply - @_retry_overload def command( self, command: Union[str, MutableMapping[str, Any]], @@ -935,12 +932,15 @@ def command( if read_preference is None: read_preference = (session and session._txn_read_preference()) or ReadPreference.PRIMARY - with self._client._conn_for_reads(read_preference, session, operation=command_name) as ( - connection, - read_preference, - ): + + def inner( + session: Optional[ClientSession], + _server: Server, + conn: Connection, + read_preference: _ServerMode, + ) -> Union[dict[str, Any], _CodecDocumentType]: return self._command( - connection, + conn, command, value, check, @@ -951,8 +951,11 @@ def command( **kwargs, ) + return self._client._retryable_read( + inner, read_preference, session, command_name, None, False + ) + @_csot.apply - @_retry_overload def cursor_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1019,15 +1022,17 @@ def cursor_command( with self._client._tmp_session(session) as tmp_session: opts = codec_options or DEFAULT_CODEC_OPTIONS - if read_preference is None: read_preference = ( tmp_session and tmp_session._txn_read_preference() ) or ReadPreference.PRIMARY - with self._client._conn_for_reads(read_preference, tmp_session, command_name) as ( - conn, - read_preference, - ): + + def inner( + session: Optional[ClientSession], + _server: Server, + conn: Connection, + read_preference: _ServerMode, + ) -> CommandCursor[_DocumentType]: response = self._command( conn, command, @@ -1036,7 +1041,7 @@ def cursor_command( None, read_preference, opts, - session=tmp_session, + session=session, **kwargs, ) coll = self.get_collection("$cmd", read_preference=read_preference) @@ -1046,7 +1051,7 @@ def cursor_command( response["cursor"], conn.address, max_await_time_ms=max_await_time_ms, - session=tmp_session, + session=session, comment=comment, ) cmd_cursor._maybe_pin_connection(conn) @@ -1054,6 +1059,10 @@ def cursor_command( else: raise InvalidOperation("Command does not return a cursor.") + return self.client._retryable_read( + inner, read_preference, tmp_session, command_name, False + ) + def _retryable_read_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1252,9 +1261,11 @@ def _drop_helper( if comment is not None: command["comment"] = comment - with self._client._conn_for_writes(session, operation=_Op.DROP) as connection: + def inner( + session: Optional[ClientSession], conn: Connection, _retryable_write: bool + ) -> dict[str, Any]: return self._command( - connection, + conn, command, allowable_errors=["ns not found", 26], write_concern=self._write_concern_for(session), @@ -1262,8 +1273,9 @@ def _drop_helper( session=session, ) + return self.client._retryable_write(False, inner, session, _Op.DROP) + @_csot.apply - @_retry_overload def drop_collection( self, name_or_collection: Union[str, Collection[_DocumentTypeArg]], diff --git a/pymongo/synchronous/helpers.py b/pymongo/synchronous/helpers.py index 2c57b7a597..c093b379ea 100644 --- a/pymongo/synchronous/helpers.py +++ b/pymongo/synchronous/helpers.py @@ -30,7 +30,6 @@ from pymongo import _csot from pymongo.errors import ( OperationFailure, - PyMongoError, ) from pymongo.helpers_shared import _REAUTHENTICATION_REQUIRED_CODE from pymongo.lock import _create_lock @@ -164,34 +163,6 @@ def should_retry(self, attempt: int, delay: float) -> bool: return True -def _retry_overload(func: F) -> F: - @functools.wraps(func) - def inner(self: Any, *args: Any, **kwargs: Any) -> Any: - retry_policy = self._retry_policy - attempt = 0 - while True: - try: - res = func(self, *args, **kwargs) - retry_policy.record_success(retry=attempt > 0) - return res - except PyMongoError as exc: - if not exc.has_error_label("RetryableError"): - raise - attempt += 1 - delay = 0 - if exc.has_error_label("SystemOverloadedError"): - delay = retry_policy.backoff(attempt) - if not retry_policy.should_retry(attempt, delay): - raise - - # Implement exponential backoff on retry. - if delay: - time.sleep(delay) - continue - - return cast(F, inner) - - def _getaddrinfo( host: Any, port: Any, **kwargs: Any ) -> list[ diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 30b8c4fc61..acf5e6b98f 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -112,7 +112,6 @@ from pymongo.synchronous.client_session import _EmptyServerSession from pymongo.synchronous.command_cursor import CommandCursor from pymongo.synchronous.helpers import ( - _retry_overload, _RetryPolicy, _TokenBucket, ) @@ -2393,7 +2392,6 @@ def list_database_names( return [doc["name"] for doc in res] @_csot.apply - @_retry_overload def drop_database( self, name_or_database: Union[str, database.Database[_DocumentTypeArg]], @@ -2436,15 +2434,13 @@ def drop_database( f"name_or_database must be an instance of str or a Database, not {type(name)}" ) - with self._conn_for_writes(session, operation=_Op.DROP_DATABASE) as conn: - self[name]._command( - conn, - {"dropDatabase": 1, "comment": comment}, - read_preference=ReadPreference.PRIMARY, - write_concern=self._write_concern_for(session), - parse_write_concern_error=True, - session=session, - ) + return self[name].command( + {"dropDatabase": 1, "comment": comment}, + read_preference=ReadPreference.PRIMARY, + write_concern=self._write_concern_for(session), + parse_write_concern_error=True, + session=session, + ) @_csot.apply def bulk_write( From 49f605a9efb185dce0ba1925c6777f565ffd5937 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Dec 2025 14:42:16 -0600 Subject: [PATCH 12/12] return type fixes --- pymongo/asynchronous/collection.py | 8 ++++---- pymongo/asynchronous/database.py | 2 +- pymongo/asynchronous/mongo_client.py | 2 +- pymongo/synchronous/collection.py | 4 ++-- pymongo/synchronous/database.py | 2 +- pymongo/synchronous/mongo_client.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index 60755dbe89..0eeff7a043 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -2786,7 +2786,7 @@ async def inner( ) return [index["name"] for index in resp["indexesCreated"]] - return self.database.client._retryable_write( + return await self.database.client._retryable_write( False, inner, session, _Op.CREATE_SEARCH_INDEXES ) @@ -2828,7 +2828,7 @@ async def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + await self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) async def update_search_index( self, @@ -2870,7 +2870,7 @@ async def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + await self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) async def options( self, @@ -3165,7 +3165,7 @@ async def inner( client=client, ) - return client._retryable_write(False, inner, session, _Op.RENAME) + return await client._retryable_write(False, inner, session, _Op.RENAME) async def distinct( self, diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 7ad9017f70..3af77ffe15 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -1060,7 +1060,7 @@ async def inner( raise InvalidOperation("Command does not return a cursor.") return await self.client._retryable_read( - inner, read_preference, tmp_session, command_name, False + inner, read_preference, tmp_session, command_name, None, False ) async def _retryable_read_command( diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 7e9b5c9a9d..d3e8e64ee9 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2444,7 +2444,7 @@ async def drop_database( f"name_or_database must be an instance of str or a AsyncDatabase, not {type(name)}" ) - return await self[name].command( + await self[name].command( {"dropDatabase": 1, "comment": comment}, read_preference=ReadPreference.PRIMARY, write_concern=self._write_concern_for(session), diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 6444e75ad8..f7da050f4c 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -2823,7 +2823,7 @@ def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) def update_search_index( self, @@ -2865,7 +2865,7 @@ def inner( session=session, ) - return self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) def options( self, diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 4fa8f4f1b4..60a55276c6 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -1060,7 +1060,7 @@ def inner( raise InvalidOperation("Command does not return a cursor.") return self.client._retryable_read( - inner, read_preference, tmp_session, command_name, False + inner, read_preference, tmp_session, command_name, None, False ) def _retryable_read_command( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index acf5e6b98f..a9723fd087 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2434,7 +2434,7 @@ def drop_database( f"name_or_database must be an instance of str or a Database, not {type(name)}" ) - return self[name].command( + self[name].command( {"dropDatabase": 1, "comment": comment}, read_preference=ReadPreference.PRIMARY, write_concern=self._write_concern_for(session),