PYTHON-2366 Test OCSP+FLE with Python 3.9 (#534)

PYTHON-2449 Move all pypy cryptography/pyopenssl testing to Debian 9.2 with OpenSSL 1.1.0f
PYTHON-2449 Fix Windows cryptography installation by upgrading pip and using --prefer-binary

(cherry picked from commit 3ecd947)

Author: ShaneHarvey

.evergreen/config.yml

@@ -1561,6 +1561,7 @@ axes:
         run_on: debian92-test
         batchtime: 10080  # 7 days
         variables:
+          python3_binary: "/opt/python/3.8/bin/python3"
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian92/master/latest/libmongocrypt.tar.gz
       - id: macos-1014
         display_name: "macOS 10.14"
@@ -2160,14 +2161,14 @@ buildvariants:
 - matrix_name: "tests-pyopenssl"
   matrix_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3.5"]
+    python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9"]
     auth: "*"
     ssl: "ssl"
     pyopenssl: "*"
   # Only test "noauth" with Python 3.7.
   exclude_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.4", "3.5", "3.6", "3.8", "3.9", "pypy", "pypy3.5"]
+    python-version: ["2.7", "3.4", "3.5", "3.6", "3.8", "3.9"]
     auth: "noauth"
     ssl: "ssl"
     pyopenssl: "*"
@@ -2177,6 +2178,19 @@ buildvariants:
     # Test standalone and sharded only on 4.4.
     - '.4.4'
 
+- matrix_name: "tests-pyopenssl-pypy"
+  matrix_spec:
+    platform: debian92
+    python-version: ["pypy", "pypy3.5", "pypy3.6"]
+    auth: "auth"
+    ssl: "ssl"
+    pyopenssl: "*"
+  display_name: "PyOpenSSL ${platform} ${python-version} ${auth}"
+  tasks:
+    - '.replica_set !.2.6 !.3.0 !.3.2 !.3.4'
+    # Test standalone and sharded only on 4.4.
+    - '.4.4'
+
 - matrix_name: "test-pyopenssl-old-py27"
   matrix_spec:
     platform:
@@ -2214,7 +2228,7 @@ buildvariants:
   matrix_spec:
     platform: rhel62
     # RHEL 6.2 does not support Python 3.7.x and later.
-    python-version: ["2.7", "3.4", "3.5", "3.6", "pypy", "pypy3.5", "pypy3.6"]
+    python-version: ["2.7", "3.4", "3.5", "3.6"]
     auth-ssl: noauth-nossl
 # TODO: dependency error for 'coverage-report' task:
 # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config
@@ -2223,6 +2237,15 @@ buildvariants:
   display_name: "Encryption ${python-version} ${platform} ${auth-ssl}"
   tasks: *encryption-server-versions
 
+- matrix_name: "tests-pypy-debian-test-encryption"
+  matrix_spec:
+    platform: debian92
+    python-version: ["pypy", "pypy3.5", "pypy3.6"]
+    auth-ssl: noauth-nossl
+    encryption: "*"
+  display_name: "Encryption ${python-version} ${platform} ${auth-ssl}"
+  tasks: *encryption-server-versions
+
 - matrix_name: "tests-python-version-rhel62-without-c-extensions"
   matrix_spec:
     platform: rhel62
@@ -2365,12 +2388,6 @@ buildvariants:
     python-version-windows: "*"
     auth-ssl: "*"
     encryption: "*"
-  exclude_spec:
-    # PYTHON-2366 Skip 3.9 due to cryptography install failures
-    - platform: "*"
-      python-version-windows: ["3.9"]
-      auth-ssl: "*"
-      encryption: "*"
   display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl}"
   tasks: *encryption-server-versions
 
@@ -2533,7 +2550,19 @@ buildvariants:
 - matrix_name: "ocsp-test"
   matrix_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.4", "3.8", "3.9", "pypy", "pypy3.5"]
+    python-version: ["2.7", "3.4", "3.8", "3.9"]
+    mongodb-version: ["4.4", "latest"]
+    auth: "noauth"
+    ssl: "ssl"
+  display_name: "OCSP test ${platform} ${python-version} ${mongodb-version}"
+  batchtime: 20160 # 14 days
+  tasks:
+    - name: ".ocsp"
+
+- matrix_name: "ocsp-test-pypy"
+  matrix_spec:
+    platform: debian92
+    python-version: ["pypy", "pypy3.5", "pypy3.6"]
     mongodb-version: ["4.4", "latest"]
     auth: "noauth"
     ssl: "ssl"
@@ -2545,7 +2574,7 @@ buildvariants:
 - matrix_name: "ocsp-test-windows"
   matrix_spec:
     platform: windows-64-vsMulti-small
-    python-version-windows: ["2.7", "3.4", "3.8"]
+    python-version-windows: ["2.7", "3.4", "3.9"]
     mongodb-version: ["4.4", "latest"]
     auth: "noauth"
     ssl: "ssl"

.evergreen/run-ocsp-tests.sh

@@ -3,6 +3,9 @@
 set -o xtrace
 set -o errexit
 
+# For createvirtualenv.
+. .evergreen/utils.sh
+
 if [ -z "$PYTHON_BINARY" ]; then
     echo "No python binary specified"
     PYTHON=$(command -v python || command -v python3) || true
@@ -14,36 +17,9 @@ else
     PYTHON="$PYTHON_BINARY"
 fi
 
-if $PYTHON -m virtualenv --version; then
-    VIRTUALENV="$PYTHON -m virtualenv"
-elif command -v virtualenv; then
-    # We can remove this fallback after:
-    # https://github.com/10gen/mongo-python-toolchain/issues/8
-    VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-else
-    echo "Cannot test without virtualenv"
-    exit 1
-fi
-
-$VIRTUALENV --never-download --no-wheel ocsptest
-if [ "Windows_NT" = "$OS" ]; then
-  . ocsptest/Scripts/activate
-else
-  . ocsptest/bin/activate
-fi
+createvirtualenv $PYTHON ocsptest
 trap "deactivate; rm -rf ocsptest" EXIT HUP
 
-IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
-if [ $IS_PYTHON_2 = "1" ]; then
-    echo "Using a Python 2"
-    # Upgrade pip to install the cryptography wheel and not the tar.
-    # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-    python -m pip install --upgrade 'pip<20.1'
-    # Upgrade setuptools because cryptography requires 18.5+.
-    # <45 because 45.0 dropped support for 2.7.
-    python -m pip install --upgrade 'setuptools<45'
-fi
-
-python -m pip install pyopenssl requests service_identity
+python -m pip install --prefer-binary pyopenssl requests service_identity
 
 OCSP_TLS_SHOULD_SUCCEED=${OCSP_TLS_SHOULD_SUCCEED} CA_FILE=${CA_FILE} python test/ocsp/test_ocsp.py

.evergreen/run-tests.sh

@@ -94,38 +94,11 @@ fi
 
 # PyOpenSSL test setup.
 if [ -n "$TEST_PYOPENSSL" ]; then
-    if $PYTHON -m virtualenv --version; then
-        VIRTUALENV="$PYTHON -m virtualenv"
-    elif command -v virtualenv; then
-        # We can remove this fallback after:
-        # https://github.com/10gen/mongo-python-toolchain/issues/8
-        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
-    else
-        echo "Cannot test without virtualenv"
-        exit 1
-    fi
-
-    $VIRTUALENV pyopenssltest
-    if [ "Windows_NT" = "$OS" ]; then
-        . pyopenssltest/Scripts/activate
-    else
-        . pyopenssltest/bin/activate
-    fi
+    createvirtualenv $PYTHON pyopenssltest
     trap "deactivate; rm -rf pyopenssltest" EXIT HUP
     PYTHON=python
 
-    IS_PYTHON_2=$(python -c "import sys; sys.stdout.write('1' if sys.version_info < (3,) else '0')")
-    if [ $IS_PYTHON_2 = "1" ]; then
-        echo "Using a Python 2"
-        # Upgrade pip to install the cryptography wheel and not the tar.
-        # <20.1 because 20.0.2 says a future release may drop support for 2.7.
-        python -m pip install --upgrade 'pip<20.1'
-        # Upgrade setuptools because cryptography requires 18.5+.
-        # <45 because 45.0 dropped support for 2.7.
-        python -m pip install --upgrade 'setuptools<45'
-    fi
-
-    python -m pip install pyopenssl requests service_identity
+    python -m pip install --prefer-binary pyopenssl requests service_identity
 fi
 
 if [ -n "$TEST_ENCRYPTION" ]; then
@@ -166,7 +139,8 @@ if [ -n "$TEST_ENCRYPTION" ]; then
 
     # TODO: Test with 'pip install pymongocrypt'
     git clone --branch master https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
-    python -m pip install --upgrade ./libmongocrypt_git/bindings/python
+    python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt
+    python -m pip install ./libmongocrypt_git/bindings/python
     python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
     python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
     # PATH is updated by PREPARE_SHELL for access to mongocryptd.

.evergreen/test-encryption-requirements.txt

@@ -0,0 +1,4 @@
+# cffi==1.14.3 was the last installable release on RHEL 6.2 with Python 3.4
+cffi==1.14.3;python_version=="3.4"
+cffi>=1.12.0,<2;python_version!="3.4"
+cryptography>=2,<4

.evergreen/utils.sh

@@ -8,19 +8,24 @@ createvirtualenv () {
     PYTHON=$1
     VENVPATH=$2
     if $PYTHON -m virtualenv --version; then
-        VIRTUALENV="$PYTHON -m virtualenv"
+        VIRTUALENV="$PYTHON -m virtualenv --never-download"
+    elif $PYTHON -m venv -h>/dev/null; then
+        VIRTUALENV="$PYTHON -m venv"
     elif command -v virtualenv; then
-        VIRTUALENV="$(command -v virtualenv) -p $PYTHON"
+        VIRTUALENV="$(command -v virtualenv) -p $PYTHON --never-download"
     else
         echo "Cannot test without virtualenv"
         exit 1
     fi
-    $VIRTUALENV --system-site-packages --never-download $VENVPATH
+    $VIRTUALENV $VENVPATH
     if [ "Windows_NT" = "$OS" ]; then
         . $VENVPATH/Scripts/activate
     else
         . $VENVPATH/bin/activate
     fi
+    # Upgrade to the latest versions of pip setuptools wheel so that
+    # pip can always download the latest cryptography+cffi wheels.
+    python -m pip install --upgrade pip setuptools wheel
 }
 
 # Usage: