From f5cecc4b4f5b18b68b746cab082698a90c3a27e9 Mon Sep 17 00:00:00 2001 From: 3manifold <22544721+3manifold@users.noreply.github.com> Date: Thu, 22 Jan 2026 09:50:47 +0100 Subject: [PATCH] Integrate Clang ThreadSanitizer in tests --- .github/workflows/ci.yml | 335 ++++++--------------------------------- CMakeLists.txt | 14 +- src/cpu/primitives.cc | 21 ++- 3 files changed, 80 insertions(+), 290 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28605553d..561d3db5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,9 @@ jobs: run: | tests/ctranslate2_test tests/data - build-and-test-cpp-x86_64-address_sanitizer: + + + build-and-test-cpp-x86_64-thread_sanitizer: runs-on: ubuntu-22.04 env: CT2_VERBOSE: 1 @@ -87,24 +89,33 @@ jobs: wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add *.PUB sudo sh -c 'echo "deb https://apt.repos.intel.com/oneapi all main" > /etc/apt/sources.list.d/oneAPI.list' - sudo apt update + sudo apt-get update - name: Install MKL env: - CT2_USE_MKL: 1 MKL_VERSION: 2023.0.0 run: | sudo apt install -y intel-oneapi-mkl-devel-$MKL_VERSION - - name: Install Clang - run: | - sudo apt install -y clang + + + - name: Install Clang2 + run: | + NPROC=$(nproc) + git clone --depth 1 --branch llvmorg-7.0.1 --recurse-submodules -j$NPROC https://github.com/llvm/llvm-project + cd llvm-project + mkdir build + cd build + cmake -DLIBOMP_TSAN_SUPPORT=1 ../openmp + sudo cmake --build . --target install --parallel $NPROC + +# sudo and --target install is optional if you adjust the path to libomp.so below - - name: Configure with MKL and Clang + - name: Configure with MKL env: CT2_USE_MKL: 1 run: | - cmake -DCMAKE_INSTALL_PREFIX=$PWD/install -DBUILD_TESTS=ON -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DENABLE_ADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug . + cmake -DCMAKE_INSTALL_PREFIX=$PWD/install -DBUILD_TESTS=ON -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DENABLE_THREAD_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug . - name: Build run: | @@ -117,306 +128,60 @@ jobs: wget https://opennmt-models.s3.amazonaws.com/transliteration-aren-all.tar.gz tar xf transliteration-aren-all.tar.gz - - name: Test AddressSanitizer + - name: Test ThreadSanitizer env: CT2_USE_MKL: 1 run: | - ASAN_OPTIONS=detect_leaks=1:print_stats=1 tests/ctranslate2_test tests/data - - build-and-test-cpp-arm64: - runs-on: ${{ matrix.os }} + ldd /usr/local/lib/libomp.so + LD_PRELOAD=/usr/local/lib/libomp.so TSAN_OPTIONS=halt_on_error=1 tests/ctranslate2_test tests/data + + build-and-test-cpp-x86_64-thread_sanitizer_NOLLVMOPENMP: + runs-on: ubuntu-22.04 env: CT2_VERBOSE: 1 - strategy: - matrix: - include: - - os: ubuntu-22.04-arm - backend: openblas - - os: macos-15 - backend: ruy steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Build with OpenBLAS and Ruy - if: matrix.backend == 'openblas' - run: | - wget https://github.com/xianyi/OpenBLAS/archive/v0.3.13.tar.gz - tar xzvf v0.3.13.tar.gz - cd OpenBLAS-0.3.13 - make TARGET=ARMV8 NO_LAPACK=1 -j $(nproc) - sudo make PREFIX=/usr/local install -j $(nproc) - cd .. - export OpenBLAS_HOME=/usr/local - cmake \ - -DOPENMP_RUNTIME=COMP \ - -DCMAKE_INSTALL_PREFIX=$PWD/install \ - -DWITH_MKL=OFF \ - -DWITH_OPENBLAS=ON \ - -DWITH_RUY=ON \ - -DBUILD_TESTS=ON \ - . - make -j $(nproc) install - - - name: Build Ruy - if: matrix.backend == 'ruy' - run: | - CMAKE_EXTRA_OPTIONS='-DCMAKE_OSX_ARCHITECTURES=arm64 -DWITH_ACCELERATE=ON -DWITH_MKL=OFF -DOPENMP_RUNTIME=NONE -DWITH_RUY=ON' - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=$PWD/install \ - -DBUILD_TESTS=ON \ - $CMAKE_EXTRA_OPTIONS \ - . - make -j $(nproc) install - - - name: Download test data - run: | - wget https://opennmt-models.s3.amazonaws.com/transliteration-aren-all.tar.gz - tar xf transliteration-aren-all.tar.gz -C tests/data/models/ - - - name: Test + - name: Install Intel oneAPI run: | - tests/ctranslate2_test tests/data - - build-python-wheels: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-22.04, windows-2022] - arch: [auto64] - include: - - os: ubuntu-22.04 - arch: aarch64 - - os: macos-15 - arch: arm64 - - os: macos-15-intel - arch: x86_64 - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: docker/setup-qemu-action@v2 - if: ${{ matrix.arch == 'aarch64' }} - name: Set up QEMU + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo apt-key add *.PUB + sudo sh -c 'echo "deb https://apt.repos.intel.com/oneapi all main" > /etc/apt/sources.list.d/oneAPI.list' + sudo apt-get update - - name: Build wheels - uses: pypa/cibuildwheel@v3.2.1 - with: - package-dir: python - output-dir: python/wheelhouse + - name: Install MKL env: - CIBW_ENVIRONMENT_PASS_LINUX: CIBW_ARCHS - CIBW_ENVIRONMENT_WINDOWS: CTRANSLATE2_ROOT='${{ github.workspace }}\install' - CIBW_ENVIRONMENT_MACOS: "CTRANSLATE2_ROOT='/usr/local' MACOSX_DEPLOYMENT_TARGET=11.00" - CIBW_BEFORE_ALL_LINUX: python/tools/prepare_build_environment_linux.sh - CIBW_BEFORE_ALL_MACOS: python/tools/prepare_build_environment_macos.sh - CIBW_BEFORE_ALL_WINDOWS: bash python/tools/prepare_build_environment_windows.sh - CIBW_BEFORE_BUILD: pip install -r python/install_requirements.txt - CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 - CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 - CIBW_ARCHS: ${{ matrix.arch }} - CIBW_SKIP: "*-musllinux_*" - - - name: Upload Python wheels - uses: actions/upload-artifact@v4 - with: - name: python-wheels-${{ runner.os }}-${{ matrix.arch }} - path: python/wheelhouse - - - # We could test the Python wheels using cibuildwheel but we prefer to run the tests outside - # the build environment to ensure wheels correctly embed all dependencies. - test-python-wheels: - needs: [build-python-wheels] - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-22.04 - artifact_pattern: python-wheels-Linux-auto64 - wheel_pattern: "*cp310*manylinux*x86_64.whl" - - - os: ubuntu-22.04-arm - artifact_pattern: python-wheels-Linux-aarch64 - wheel_pattern: "*cp310*manylinux*_aarch64.whl" - - - os: macos-15 - artifact_pattern: python-wheels-macOS-arm64 - wheel_pattern: "*cp310*macosx*arm64.whl" - - steps: - - name: Set up Python 3.10.11 - uses: actions/setup-python@v5 - with: - python-version: "3.10.11" - - - uses: actions/checkout@v4 - - - name: Prepare test environment - shell: bash - run: | - ./python/tools/prepare_test_environment.sh - - - name: Download Python wheels - uses: actions/download-artifact@v4 - with: - pattern: ${{ matrix.artifact_pattern }} - merge-multiple: true - path: . - - - name: Install wheel - shell: bash - run: | - pip install ${{ matrix.wheel_pattern }} - - - name: Test Python wheel - run: | - pytest -v python/tests/ --ignore=python/tests/test_opennmt_tf.py - - - check-python-style: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.10.11 - uses: actions/setup-python@v5 - with: - python-version: "3.10.11" - - - name: Install dependencies - run: | - python -m pip install black==22.* flake8==3.8.* isort==5.* - - - name: Check code format with Black - working-directory: python - run: | - black --check . - - - name: Check imports order with isort - working-directory: python - run: | - isort --check-only . - - - name: Check code style with Flake8 - working-directory: python - if: ${{ always() }} - run: | - flake8 . - - - publish-python-wheels-on-pypi: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - needs: [build-and-test-cpp-x86_64, build-and-test-cpp-arm64, build-python-wheels, test-python-wheels, check-python-style] - runs-on: ubuntu-22.04 - - steps: - - name: Download Python wheels - uses: actions/download-artifact@v4 - with: - pattern: python-wheels-* - merge-multiple: true - path: . - - - name: Publish Python wheels to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: . - - - build-and-push-docker-images: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Show disk and docker usage (before cleanup) - run: | - df -h - echo " -= Docker System =-" - docker system df || true - - - name: Free disk space (cleanup heavy preinstalled directories + docker prune) - run: | - echo " -= Removing big preinstalled directories (shouldn't remove the needed tools) =-" - sudo rm -rf /opt/hostedtoolcache || true - sudo rm -rf /usr/share/dotnet || true - sudo rm -rf /usr/lib/jvm || true - sudo rm -rf /usr/local/lib/android || true - echo " -= Running docker prune =-" - docker system prune -af --volumes || true - docker builder prune -af || true - - - name: Show disk and docker usage (after cleanup) + MKL_VERSION: 2023.0.0 run: | - df -h - echo " -= Docker System =-" - docker system df || true + sudo apt install -y intel-oneapi-mkl-devel-$MKL_VERSION - - name: Build Docker images + - name: Install Clang run: | - ./docker/build_all.sh - - - name: Login to DockerHub - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + sudo apt install -y clang - - name: Push Docker images - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + - name: Configure with MKL + env: + CT2_USE_MKL: 1 run: | - ./docker/build_all.sh ${GITHUB_REF##*/v} 1 - - - build-and-deploy-docs: - runs-on: ubuntu-latest - needs: [check-python-style, build-python-wheels] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.10.11 - uses: actions/setup-python@v5 - with: - python-version: "3.10.11" + cmake -DCMAKE_INSTALL_PREFIX=$PWD/install -DBUILD_TESTS=ON -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ -DENABLE_THREAD_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug . - - name: Download CTranslate2 wheels - uses: actions/download-artifact@v4 - with: - pattern: python-wheels-${{ runner.os }}-* - merge-multiple: true - path: . - - - name: Install CTranslate2 wheel + - name: Build run: | - pip install *cp310*manylinux*x86_64.whl + make -j $(nproc) install - - name: Install dependencies to build docs - working-directory: docs + - name: Download test data + working-directory: tests/data/models run: | - python -m pip install -r requirements.txt + wget https://opennmt-models.s3.amazonaws.com/pi_lm_step_5000.pt + wget https://opennmt-models.s3.amazonaws.com/transliteration-aren-all.tar.gz + tar xf transliteration-aren-all.tar.gz - - name: Build docs - working-directory: docs + - name: Test ThreadSanitizer + env: + CT2_USE_MKL: 1 run: | - python generate.py python - sphinx-build . build + TSAN_OPTIONS=halt_on_error=1 tests/ctranslate2_test tests/data - - name: Deploy docs - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: docs/build - clean: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 0892d6b00..1c3d784aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(WITH_TENSOR_PARALLEL "Compile with NCCL and MPI backend" OFF) option(WITH_FLASH_ATTN "Compile with Flash Attention 2" OFF) option(ENABLE_ADDRESS_SANITIZER "ASAN" OFF) +option(ENABLE_THREAD_SANITIZER "TSAN" OFF) MESSAGE(STATUS "Compiler Id: ${CMAKE_CXX_COMPILER_ID}") MESSAGE(STATUS "Compiler Version: ${CMAKE_CXX_COMPILER_VERSION}") @@ -480,13 +481,20 @@ if (WITH_RUY) list(APPEND LIBRARIES ruy) endif() +# sanitizers IF (ENABLE_ADDRESS_SANITIZER AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "DEBUG")) - MESSAGE (STATUS "ENABLE_ADDRESS_SANITIZER: ENABLED") - set(ASAN_FLAGS " -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-common") + MESSAGE (STATUS "ENABLE_ADDRESS_SANITIZER: TRUE") + set(ASAN_FLAGS " -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-common ") string(APPEND CMAKE_C_FLAGS ${ASAN_FLAGS}) string(APPEND CMAKE_CXX_FLAGS ${ASAN_FLAGS}) add_link_options(-fsanitize=address) -ELSEIF (ENABLE_ADDRESS_SANITIZER) +ELSEIF (ENABLE_THREAD_SANITIZER AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "DEBUG")) + MESSAGE (STATUS "ENABLE_THREAD_SANITIZER: TRUE") + set(TSAN_FLAGS " -g -fsanitize=thread") + string(APPEND CMAKE_C_FLAGS ${TSAN_FLAGS}) + string(APPEND CMAKE_CXX_FLAGS ${TSAN_FLAGS}) + add_link_options(-fsanitize=thread) +ELSEIF (ENABLE_ADDRESS_SANITIZER OR ENABLE_THREAD_SANITIZER) MESSAGE(FATAL_ERROR "SANITIZER requires Debug build type") ENDIF () diff --git a/src/cpu/primitives.cc b/src/cpu/primitives.cc index 3f910c659..601a901bc 100644 --- a/src/cpu/primitives.cc +++ b/src/cpu/primitives.cc @@ -294,12 +294,29 @@ namespace ctranslate2 { template<> template<> void primitives::relu(const float* x, float* y, dim_t size) { + + int ssum = 0; + int n = 10000; + +#pragma omp parallel for reduction(+:ssum) + for (int i = 0; i < n; i++) { + ssum += i; // Each thread has private copy, combined at end + } + + + float sum = 0.0f; // Shared state + cpu::parallel_for(0, size, cpu::GRAIN_SIZE, - [x, y](dim_t begin, dim_t end) { + [x, y, &sum](dim_t begin, dim_t end) { max(float(0), x + begin, y + begin, end - begin); + // DATA RACE: multiple threads may write to y[0] + // y[0] = x[0] > 0 ? x[0] : 0; + // DATA RACE: unsynchronized read-modify-write + for (dim_t i = begin; i < end; ++i) { + sum += y[i]; + } }); } - template<> template<> void primitives::gelu(const float* x, float* y, dim_t size) {