From d1c47f4304ac992c0a46ea6b77b4c7093b694371 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Mon, 18 May 2026 17:16:31 +0800 Subject: [PATCH] [Velox] Fix non-deterministic native build by isolating /usr/local on macOS On Homebrew-based macOS, `/usr/local` accumulates leftover headers, libraries, and CMake packages from prior Velox/Gluten builds (often root-owned after past `sudo make install` runs from `cmake_install` in the bootstrap script). When the native build includes `/usr/local/include` and `/usr/local/lib/cmake` in its CMake search paths it picks up stale or mismatched versions of re2, glog, fmt, etc., shadowing Velox's `deps-install` copies and producing confusing link or compile errors. Concrete failure modes seen: - Fresh build picks up `/usr/local/lib/cmake/glog` from a prior `cmake --install` and links a stale glog into libvelox.dylib alongside the bundled glog from `deps-install`. - `/usr/local/include` shadows Cellar headers (e.g. fmt) and yields `static_assert` failures because Velox's bundled fmt advanced past what `/usr/local/include/fmt` provides. - Link errors against an old re2 left over from an aborted Velox bootstrap. Force CMake to ignore `/usr/local` when building native deps on macOS in three places that drive third-party / Velox / Gluten CMake configs: - `dev/build-helper-functions.sh::cmake_install`: adds `-DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON`, `-DCMAKE_IGNORE_PREFIX_PATH=/usr/local`, `-DCMAKE_IGNORE_PATH`, `-DCMAKE_SYSTEM_IGNORE_PATH` for the third-party CMake invocations driven by the Arrow/Velox bootstrap. - `ep/build-velox/src/build-velox.sh::compile`: same flags when configuring Velox itself, plus suppress two Apple-Clang-only warnings that fire under `-Werror` on macOS but are benign noise: `-Wno-inconsistent-missing-override` and `-Wno-macro-redefined`. - `dev/builddeps-veloxbe.sh::build_gluten_cpp`: same isolation flags plus the same warning suppressions for the Gluten CMake configure. All blocks are gated on `[[ "$(uname)" == "Darwin" ]]` / `if [ $OS == 'Darwin' ]`. Linux paths are unchanged. Without this isolation, the macOS native build is non-deterministic depending on what's left over in `/usr/local`; with it the build only reads from the well-defined Homebrew Cellar prefix (`/opt/homebrew`) and Velox's `deps-install`. Verification: - macOS 14 arm64 with Apple Clang 17 and a `/usr/local` populated with stale glog/re2/fmt installs from a prior session: with these flags applied, the build no longer references `/usr/local/include` or `/usr/local/lib` in any compile or link command (verified via `_build/release/build.ninja`), and produces clean libvelox.dylib + libgluten.dylib that pass the gluten cpp ctest matrix. - Linux x86_64 build green; all blocks are Darwin-gated. --- dev/build-helper-functions.sh | 9 +++++++++ dev/builddeps-veloxbe.sh | 5 +++++ ep/build-velox/src/build-velox.sh | 9 +++++++++ 3 files changed, 23 insertions(+) diff --git a/dev/build-helper-functions.sh b/dev/build-helper-functions.sh index 4be8ad170ba..210052cf9dc 100644 --- a/dev/build-helper-functions.sh +++ b/dev/build-helper-functions.sh @@ -176,6 +176,14 @@ function cmake_install { CPU_TARGET="${CPU_TARGET:-unknown}" COMPILER_FLAGS=$(get_cxx_flags $CPU_TARGET) + local MACOS_ISOLATION_FLAGS="" + if [[ "$(uname)" == "Darwin" ]]; then + MACOS_ISOLATION_FLAGS="-DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON \ + -DCMAKE_IGNORE_PREFIX_PATH=/usr/local \ + -DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake \ + -DCMAKE_SYSTEM_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake" + fi + # CMAKE_POSITION_INDEPENDENT_CODE is required so that Velox can be built into dynamic libraries \ cmake -Wno-dev -B"${BINARY_DIR}" \ -GNinja \ @@ -185,6 +193,7 @@ function cmake_install { "${INSTALL_PREFIX+-DCMAKE_INSTALL_PREFIX=}${INSTALL_PREFIX-}" \ -DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" \ -DBUILD_TESTING=OFF \ + $MACOS_ISOLATION_FLAGS \ "$@" cmake --build "${BINARY_DIR}" diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 6ce2c6d0831..ddc68e0a534 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -259,6 +259,11 @@ function build_gluten_cpp { if [ $OS == 'Darwin' ]; then GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" + GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON" + GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_IGNORE_PREFIX_PATH=/usr/local" + GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake" + GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_SYSTEM_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake" + GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_CXX_FLAGS=-Wno-inconsistent-missing-override -Wno-macro-redefined" fi cmake -G Ninja $GLUTEN_CMAKE_OPTIONS .. diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index 99b066415b9..78f3d969c6d 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -103,10 +103,19 @@ function compile { # or error occurs. CXX_FLAGS='-Wno-error=stringop-overflow -Wno-error=cpp -Wno-missing-field-initializers \ -Wno-error=uninitialized -Wno-unknown-warning-option -Wno-deprecated-declarations' + if [[ "$(uname)" == "Darwin" ]]; then + CXX_FLAGS="$CXX_FLAGS -Wno-inconsistent-missing-override -Wno-macro-redefined" + fi COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF \ -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \ -DVELOX_ENABLE_GEO=OFF" + if [[ "$(uname)" == "Darwin" ]]; then + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON" + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local" + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake" + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_SYSTEM_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake" + fi if [ $BUILD_TEST_UTILS == "ON" ]; then COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON" fi