PYTHON-5683: Spike: Investigate using Rust for Extension Modules

- Implement comprehensive Rust BSON encoder/decoder
- Add Evergreen CI configuration and test scripts
- Add GitHub Actions workflow for Rust testing
- Add runtime selection via PYMONGO_USE_RUST environment variable
- Add performance benchmarking suite
- Update build system to support Rust extension
- Add documentation for Rust extension usage and testing"

Author: aclark4life

.evergreen/generated_configs/functions.yml

@@ -111,6 +111,8 @@ functions:
           - LOAD_BALANCER
           - LOCAL_ATLAS
           - NO_EXT
+          - PYMONGO_BUILD_RUST
+          - PYMONGO_USE_RUST
       type: test
     - command: expansions.update
       params:
@@ -152,6 +154,8 @@ functions:
           - IS_WIN32
           - REQUIRE_FIPS
           - TEST_MIN_DEPS
+          - PYMONGO_BUILD_RUST
+          - PYMONGO_USE_RUST
       type: test
     - command: subprocess.exec
       params:

.evergreen/generated_configs/tasks.yml

@@ -2554,6 +2554,21 @@ tasks:
       - func: attach benchmark test results
       - func: send dashboard data
     tags: [perf]
+  - name: perf-8.0-standalone-ssl-rust
+    commands:
+      - func: run server
+        vars:
+          VERSION: v8.0-perf
+          SSL: ssl
+      - func: run tests
+        vars:
+          TEST_NAME: perf
+          SUB_TEST_NAME: rust
+          PYMONGO_BUILD_RUST: "1"
+          PYMONGO_USE_RUST: "1"
+      - func: attach benchmark test results
+      - func: send dashboard data
+    tags: [perf]
   - name: perf-8.0-standalone
     commands:
       - func: run server
@@ -2580,6 +2595,21 @@ tasks:
       - func: attach benchmark test results
       - func: send dashboard data
     tags: [perf]
+  - name: perf-8.0-standalone-rust
+    commands:
+      - func: run server
+        vars:
+          VERSION: v8.0-perf
+          SSL: nossl
+      - func: run tests
+        vars:
+          TEST_NAME: perf
+          SUB_TEST_NAME: rust
+          PYMONGO_BUILD_RUST: "1"
+          PYMONGO_USE_RUST: "1"
+      - func: attach benchmark test results
+      - func: send dashboard data
+    tags: [perf]
 
   # Search index tests
   - name: test-search-index-helpers

.evergreen/generated_configs/variants.yml

@@ -477,6 +477,40 @@ buildvariants:
     expansions:
       SUB_TEST_NAME: pyopenssl
 
+  # Rust tests
+  - name: test-with-rust-extension
+    tasks:
+      - name: .test-standard .server-latest .pr
+    display_name: Test with Rust Extension
+    run_on:
+      - rhel87-small
+    expansions:
+      PYMONGO_BUILD_RUST: "1"
+      PYMONGO_USE_RUST: "1"
+    tags: [rust, pr]
+  - name: test-with-rust-extension---macos-arm64
+    tasks:
+      - name: .test-standard .server-latest !.pr
+    display_name: Test with Rust Extension - macOS ARM64
+    run_on:
+      - macos-14-arm64
+    batchtime: 10080
+    expansions:
+      PYMONGO_BUILD_RUST: "1"
+      PYMONGO_USE_RUST: "1"
+    tags: [rust]
+  - name: test-with-rust-extension---windows
+    tasks:
+      - name: .test-standard .server-latest !.pr
+    display_name: Test with Rust Extension - Windows
+    run_on:
+      - windows-64-vsMulti-small
+    batchtime: 10080
+    expansions:
+      PYMONGO_BUILD_RUST: "1"
+      PYMONGO_USE_RUST: "1"
+    tags: [rust]
+
   # Search index tests
   - name: search-index-helpers-rhel8
     tasks:

.evergreen/scripts/generate_config.py

@@ -958,18 +958,24 @@ def create_search_index_tasks():
 
 def create_perf_tasks():
     tasks = []
-    for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async"]):
+    for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async", "rust"]):
         vars = dict(VERSION=f"v{version}-perf", SSL=ssl)
         server_func = FunctionCall(func="run server", vars=vars)
-        vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync)
-        test_func = FunctionCall(func="run tests", vars=vars)
+        test_vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync)
+        # Enable Rust for rust perf tests
+        if sync == "rust":
+            test_vars["PYMONGO_BUILD_RUST"] = "1"
+            test_vars["PYMONGO_USE_RUST"] = "1"
+        test_func = FunctionCall(func="run tests", vars=test_vars)
         attach_func = FunctionCall(func="attach benchmark test results")
         send_func = FunctionCall(func="send dashboard data")
         task_name = f"perf-{version}-standalone"
         if ssl == "ssl":
             task_name += "-ssl"
         if sync == "async":
             task_name += "-async"
+        elif sync == "rust":
+            task_name += "-rust"
         tags = ["perf"]
         commands = [server_func, test_func, attach_func, send_func]
         tasks.append(EvgTask(name=task_name, tags=tags, commands=commands))
@@ -1189,6 +1195,8 @@ def create_run_server_func():
         "LOAD_BALANCER",
         "LOCAL_ATLAS",
         "NO_EXT",
+        "PYMONGO_BUILD_RUST",
+        "PYMONGO_USE_RUST",
     ]
     args = [".evergreen/just.sh", "run-server", "${TEST_NAME}"]
     sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
@@ -1222,6 +1230,8 @@ def create_run_tests_func():
         "IS_WIN32",
         "REQUIRE_FIPS",
         "TEST_MIN_DEPS",
+        "PYMONGO_BUILD_RUST",
+        "PYMONGO_USE_RUST",
     ]
     args = [".evergreen/just.sh", "setup-tests", "${TEST_NAME}", "${SUB_TEST_NAME}"]
     setup_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
@@ -1283,6 +1293,55 @@ def create_send_dashboard_data_func():
     return "send dashboard data", cmds
 
 
+def create_rust_variants():
+    """Create build variants that test with Rust extension alongside C extension."""
+    variants = []
+
+    # Test Rust on Linux (primary platform) - runs on PRs
+    # Run standard tests with Rust enabled (both sync and async)
+    variant = create_variant(
+        [".test-standard .server-latest .pr"],
+        "Test with Rust Extension",
+        host=DEFAULT_HOST,
+        tags=["rust", "pr"],
+        expansions=dict(
+            PYMONGO_BUILD_RUST="1",
+            PYMONGO_USE_RUST="1",
+        ),
+    )
+    variants.append(variant)
+
+    # Test on macOS ARM64 (important for M1/M2 Macs)
+    variant = create_variant(
+        [".test-standard .server-latest !.pr"],
+        "Test with Rust Extension - macOS ARM64",
+        host=HOSTS["macos-arm64"],
+        tags=["rust"],
+        batchtime=BATCHTIME_WEEK,
+        expansions=dict(
+            PYMONGO_BUILD_RUST="1",
+            PYMONGO_USE_RUST="1",
+        ),
+    )
+    variants.append(variant)
+
+    # Test on Windows (important for cross-platform compatibility)
+    variant = create_variant(
+        [".test-standard .server-latest !.pr"],
+        "Test with Rust Extension - Windows",
+        host=HOSTS["win64"],
+        tags=["rust"],
+        batchtime=BATCHTIME_WEEK,
+        expansions=dict(
+            PYMONGO_BUILD_RUST="1",
+            PYMONGO_USE_RUST="1",
+        ),
+    )
+    variants.append(variant)
+
+    return variants
+
+
 mod = sys.modules[__name__]
 write_variants_to_file(mod)
 write_tasks_to_file(mod)

.evergreen/scripts/install-dependencies.sh

@@ -30,7 +30,7 @@ fi
 
 # Ensure just is installed.
 if ! command -v just &>/dev/null; then
-  uv tool install rust-just
+  uv tool install rust-just || uv tool install --force rust-just
 fi
 
 popd > /dev/null

.evergreen/scripts/install-rust.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Install Rust toolchain for building the Rust BSON extension.
+set -eu
+
+echo "Installing Rust toolchain..."
+
+# Check if Rust is already installed
+if command -v cargo &> /dev/null; then
+    echo "Rust is already installed:"
+    rustc --version
+    cargo --version
+    echo "Updating Rust toolchain..."
+    rustup update stable
+else
+    echo "Rust not found. Installing Rust..."
+
+    # Install Rust using rustup
+    if [ "Windows_NT" = "${OS:-}" ]; then
+        # Windows installation
+        curl --proto '=https' --tlsv1.2 -sSf https://win.rustup.rs/x86_64 -o rustup-init.exe
+        ./rustup-init.exe -y --default-toolchain stable
+        rm rustup-init.exe
+
+        # Add to PATH for current session
+        export PATH="$HOME/.cargo/bin:$PATH"
+    else
+        # Unix-like installation (Linux, macOS)
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+
+        # Source cargo env
+        source "$HOME/.cargo/env"
+    fi
+
+    echo "Rust installation complete:"
+    rustc --version
+    cargo --version
+fi
+
+# Install maturin if not already installed
+if ! command -v maturin &> /dev/null; then
+    echo "Installing maturin..."
+    cargo install maturin
+    echo "maturin installation complete:"
+    maturin --version
+else
+    echo "maturin is already installed:"
+    maturin --version
+fi
+
+echo "Rust toolchain setup complete."

.evergreen/scripts/run_tests.py

@@ -151,6 +151,16 @@ def run() -> None:
     if os.environ.get("PYMONGOCRYPT_LIB"):
         handle_pymongocrypt()
 
+    # Check if Rust extension is being used
+    if os.environ.get("PYMONGO_USE_RUST") or os.environ.get("PYMONGO_BUILD_RUST"):
+        try:
+            import bson
+
+            LOGGER.info(f"BSON implementation: {bson.get_bson_implementation()}")
+            LOGGER.info(f"Has Rust: {bson.has_rust()}, Has C: {bson.has_c()}")
+        except Exception as e:
+            LOGGER.warning(f"Could not check BSON implementation: {e}")
+
     LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}")
 
     # Record the start time for a perf test.

.evergreen/scripts/setup-dev-env.sh

@@ -22,6 +22,11 @@ bash $HERE/install-dependencies.sh
 # Handle the value for UV_PYTHON.
 . $HERE/setup-uv-python.sh
 
+# Show Rust toolchain status for debugging
+echo "Rust toolchain: $(rustc --version 2>/dev/null || echo 'not found')"
+echo "Cargo: $(cargo --version 2>/dev/null || echo 'not found')"
+echo "Maturin: $(maturin --version 2>/dev/null || echo 'not found')"
+
 # Only run the next part if not running on CI.
 if [ -z "${CI:-}" ]; then
   # Add the default install path to the path if needed.

.evergreen/scripts/setup-tests.sh

@@ -13,6 +13,8 @@ set -eu
 #  MONGODB_API_VERSION  The mongodb api version to use in tests.
 #  MONGODB_URI          If non-empty, use as the MONGODB_URI in tests.
 #  USE_ACTIVE_VENV      If non-empty, use the active virtual environment.
+#  PYMONGO_BUILD_RUST   If non-empty, build and test with Rust extension.
+#  PYMONGO_USE_RUST     If non-empty, use the Rust extension for tests.
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
 
@@ -21,6 +23,12 @@ if [ -f $SCRIPT_DIR/env.sh ]; then
   source $SCRIPT_DIR/env.sh
 fi
 
+# Install Rust toolchain if building Rust extension
+if [ -n "${PYMONGO_BUILD_RUST:-}" ]; then
+  echo "PYMONGO_BUILD_RUST is set, installing Rust toolchain..."
+  bash $SCRIPT_DIR/install-rust.sh
+fi
+
 echo "Setting up tests with args \"$*\"..."
 uv run ${USE_ACTIVE_VENV:+--active} "$SCRIPT_DIR/setup_tests.py" "$@"
 echo "Setting up tests with args \"$*\"... done."

.evergreen/scripts/setup_tests.py

@@ -32,6 +32,8 @@
     "UV_PYTHON",
     "REQUIRE_FIPS",
     "IS_WIN32",
+    "PYMONGO_USE_RUST",
+    "PYMONGO_BUILD_RUST",
 ]
 
 # Map the test name to test extra.
@@ -447,7 +449,7 @@ def handle_test_env() -> None:
 
         # PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
         # affects the benchmark results.
-        if sub_test_name == "sync":
+        if sub_test_name == "sync" or sub_test_name == "rust":
             TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}"
         else:
             TEST_ARGS = f"test/performance/async_perf_test.py {TEST_ARGS}"
@@ -471,6 +473,10 @@ def handle_test_env() -> None:
         if TEST_SUITE:
             TEST_ARGS = f"-m {TEST_SUITE} {TEST_ARGS}"
 
+        # For test_bson, run the specific test file
+        if test_name == "test_bson":
+            TEST_ARGS = f"test/test_bson.py {TEST_ARGS}"
+
     write_env("TEST_ARGS", TEST_ARGS)
     write_env("UV_ARGS", " ".join(UV_ARGS))