From d73b3a6bb3836b5f9b96ad3f355323f5366e639e Mon Sep 17 00:00:00 2001 From: zdevito Date: Mon, 8 Dec 2025 10:31:26 -0800 Subject: [PATCH] Link statically Draft of how we can link everything into monarch statically. After looking at binary size it is pretty clear we want to dynamically link libnccl (it can be big if it supports many arches). So I will change that to an approach similar to libcuda where we load the needed functions dynamically. ``` (monarch) [zdevito@devgpu014 /data/users/zdevito/fbsource/fbcode/monarch]$ ldd ./python/monarch/_rust_bindings.so linux-vdso.so.1 (0x00007ffdabdf5000) libpython3.11.so.1.0 => /home/zdevito/local/miniconda3/envs/monarch/lib/libpython3.11.so.1.0 (0x00007fe1fae00000) libgcc_s.so.1 => /home/zdevito/local/miniconda3/envs/monarch/lib/libgcc_s.so.1 (0x00007fe201a0f000) libm.so.6 => /lib64/libm.so.6 (0x00007fe201921000) libc.so.6 => /lib64/libc.so.6 (0x00007fe1faa00000) /lib64/ld-linux-x86-64.so.2 (0x00007fe201a2b000) libpthread.so.0 => /lib64/libpthread.so.0 (0x00007fe20191a000) libdl.so.2 => /lib64/libdl.so.2 (0x00007fe201915000) libutil.so.1 => /lib64/libutil.so.1 (0x00007fe201910000) ``` Differential Revision: [D88540873](https://our.internmc.facebook.com/intern/diff/D88540873/) [ghstack-poisoned] --- .gitignore | 1 + Cargo.toml | 1 + build_utils/src/lib.rs | 108 ++++++++++ monarch_cpp_static_libs/build.rs | 305 +++++++++++++++++++++++++++++ monarch_cpp_static_libs/src/lib.rs | 16 ++ monarch_extension/build.rs | 11 +- nccl-sys/Cargo.toml | 4 + nccl-sys/build.rs | 7 +- rdmaxcel-sys/Cargo.toml | 4 + rdmaxcel-sys/build.rs | 27 ++- torch-sys-cuda/build.rs | 11 +- 11 files changed, 475 insertions(+), 20 deletions(-) create mode 100644 monarch_cpp_static_libs/build.rs create mode 100644 monarch_cpp_static_libs/src/lib.rs diff --git a/.gitignore b/.gitignore index d87c0a063..5b824c6ce 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ docs/_build/** docs/build/** docs/**/generated/** */sg_execution_times.rst +feasibility diff --git a/Cargo.toml b/Cargo.toml index 55439dded..359498242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "monarch_rdma", "monarch_tensor_worker", "monarch_types", + "monarch_cpp_static_libs", "nccl-sys", "ndslice", "preempt_rwlock", diff --git a/build_utils/src/lib.rs b/build_utils/src/lib.rs index e210ad04b..a2ade5e5e 100644 --- a/build_utils/src/lib.rs +++ b/build_utils/src/lib.rs @@ -276,6 +276,114 @@ pub fn print_cuda_lib_error_help() { eprintln!("Or: export CUDA_LIB_DIR=/usr/lib64"); } +/// Emit cargo directives to statically link libstdc++ +/// +/// This finds the GCC library path containing libstdc++.a and emits the +/// appropriate cargo directives to link it statically. This avoids runtime +/// dependency on system libstdc++.so which can cause GLIBCXX version conflicts. +/// +/// Uses the `cc` crate to detect the C++ compiler, ensuring we use the same +/// compiler that `cc::Build` and `cxx_build` would use. +pub fn link_libstdcpp_static() { + // Use cc crate to get the C++ compiler, same as cc::Build and cxx_build use + let compiler = cc::Build::new().cpp(true).get_compiler(); + let gcc_lib_path = std::process::Command::new(compiler.path()) + .args(["-print-file-name=libstdc++.a"]) + .output() + .ok() + .and_then(|output| { + if output.status.success() { + String::from_utf8(output.stdout).ok().and_then(|s| { + let path = PathBuf::from(s.trim()); + path.parent().map(|p| p.to_path_buf()) + }) + } else { + None + } + }); + if let Some(gcc_lib_path) = gcc_lib_path { + println!("cargo:rustc-link-search=native={}", gcc_lib_path.display()); + } + println!("cargo:rustc-link-lib=static=stdc++"); +} + +/// Configuration for NCCL and rdma-core static libraries from nccl-static-sys. +/// +/// Use `NcclStaticConfig::from_env()` to get the paths, then use the include +/// paths for bindgen/cc, and call `emit_link_directives()` to link. +pub struct NcclStaticConfig { + pub nccl_include: String, + pub rdma_include: String, + pub nccl_lib_dir: String, + pub rdma_lib_dir: String, + pub rdma_util_dir: String, +} + +impl NcclStaticConfig { + /// Load configuration from DEP_* environment variables set by nccl-static-sys. + /// + /// The nccl-static-sys crate must be listed as a build-dependency. + pub fn from_env() -> Self { + Self { + nccl_include: std::env::var("DEP_NCCL_STATIC_NCCL_INCLUDE") + .expect("DEP_NCCL_STATIC_NCCL_INCLUDE not set - add nccl-static-sys as build-dependency"), + rdma_include: std::env::var("DEP_NCCL_STATIC_RDMA_INCLUDE") + .expect("DEP_NCCL_STATIC_RDMA_INCLUDE not set - add nccl-static-sys as build-dependency"), + nccl_lib_dir: std::env::var("DEP_NCCL_STATIC_NCCL_LIB_DIR") + .expect("DEP_NCCL_STATIC_NCCL_LIB_DIR not set - add nccl-static-sys as build-dependency"), + rdma_lib_dir: std::env::var("DEP_NCCL_STATIC_RDMA_LIB_DIR") + .expect("DEP_NCCL_STATIC_RDMA_LIB_DIR not set - add nccl-static-sys as build-dependency"), + rdma_util_dir: std::env::var("DEP_NCCL_STATIC_RDMA_UTIL_DIR") + .expect("DEP_NCCL_STATIC_RDMA_UTIL_DIR not set - add nccl-static-sys as build-dependency"), + } + } + + /// Emit all cargo link directives for static linking of NCCL and rdma-core. + /// + /// This emits search paths and link-lib directives for: + /// - libnccl_static.a + /// - libmlx5.a + /// - libibverbs.a + /// - librdma_util.a + pub fn emit_link_directives(&self) { + // Emit link search paths + println!("cargo::rustc-link-search=native={}", self.nccl_lib_dir); + println!("cargo::rustc-link-search=native={}", self.rdma_lib_dir); + println!("cargo::rustc-link-search=native={}", self.rdma_util_dir); + + // Static libraries - order matters for dependency resolution + // Use whole-archive for NCCL static library to ensure all symbols are included + println!("cargo::rustc-link-arg=-Wl,--whole-archive"); + println!("cargo::rustc-link-lib=static=nccl_static"); + println!("cargo::rustc-link-arg=-Wl,--no-whole-archive"); + + // Use whole-archive for rdma-core static libraries + println!("cargo::rustc-link-arg=-Wl,--whole-archive"); + println!("cargo::rustc-link-lib=static=mlx5"); + println!("cargo::rustc-link-lib=static=ibverbs"); + println!("cargo::rustc-link-arg=-Wl,--no-whole-archive"); + + // rdma_util helper library + println!("cargo::rustc-link-lib=static=rdma_util"); + } +} + +/// Convenience function to set up NCCL/rdma-core static linking. +/// +/// Returns the config with include paths, and emits all link directives. +/// The nccl-static-sys crate must be listed as a build-dependency. +/// +/// Example: +/// ```ignore +/// let config = build_utils::setup_nccl_static(); +/// // Use config.nccl_include and config.rdma_include for bindgen/cc +/// ``` +pub fn setup_nccl_static() -> NcclStaticConfig { + let config = NcclStaticConfig::from_env(); + config.emit_link_directives(); + config +} + #[cfg(test)] mod tests { use super::*; diff --git a/monarch_cpp_static_libs/build.rs b/monarch_cpp_static_libs/build.rs new file mode 100644 index 000000000..ad64fd774 --- /dev/null +++ b/monarch_cpp_static_libs/build.rs @@ -0,0 +1,305 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Static NCCL and rdma-core build script +//! +//! This build script: +//! 1. Copies rdma-core and NCCL from the feasibility directories +//! 2. Builds rdma-core with static libraries (libibverbs.a, libmlx5.a) +//! 3. Builds NCCL with static linking to rdma-core (libnccl_static.a) +//! 4. Emits link directives for downstream crates + +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; + +// Repository configuration +// Source: https://github.com/NVIDIA/nccl - tag v2.28.9-1 +const NCCL_SRC: &str = "feasibility/nccl"; +// Source: https://github.com/linux-rdma/rdma-core - tag v60.0 +const RDMA_CORE_SRC: &str = "feasibility/rdma-core"; + +#[cfg(target_os = "macos")] +fn main() {} + +#[cfg(not(target_os = "macos"))] +fn main() { + let manifest_dir = + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")); + let monarch_root = manifest_dir + .parent() + .expect("Failed to get monarch root directory"); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set")); + let vendor_dir = out_dir.join("vendor"); + std::fs::create_dir_all(&vendor_dir).expect("Failed to create vendor directory"); + + let rdma_core_src = monarch_root.join(RDMA_CORE_SRC); + let nccl_src = monarch_root.join(NCCL_SRC); + + let rdma_core_dir = vendor_dir.join("rdma-core"); + let nccl_dir = vendor_dir.join("nccl"); + + // Copy source directories + copy_dir(&rdma_core_src, &rdma_core_dir); + copy_dir(&nccl_src, &nccl_dir); + + // Build rdma-core first (NCCL depends on it) + let rdma_build_dir = build_rdma_core(&rdma_core_dir); + + // Build NCCL with static rdma-core + let nccl_build_dir = build_nccl(&nccl_dir, &rdma_build_dir); + + // Emit link directives + emit_link_directives(&nccl_build_dir, &rdma_build_dir); +} + +fn copy_dir(src_dir: &Path, target_dir: &Path) { + if target_dir.exists() { + println!( + "cargo:warning=Directory already exists at {}", + target_dir.display() + ); + return; + } + + println!( + "cargo:warning=Copying {} to {}", + src_dir.display(), + target_dir.display() + ); + + let status = Command::new("cp") + .args(["-r", src_dir.to_str().unwrap(), target_dir.to_str().unwrap()]) + .status() + .expect("Failed to execute cp"); + + if !status.success() { + panic!( + "Failed to copy from {} to {}", + src_dir.display(), + target_dir.display() + ); + } +} + +fn build_rdma_core(rdma_core_dir: &Path) -> PathBuf { + let build_dir = rdma_core_dir.join("build"); + + // Check if already built + if build_dir.join("lib/statics/libibverbs.a").exists() { + println!("cargo:warning=rdma-core already built"); + return build_dir; + } + + std::fs::create_dir_all(&build_dir).expect("Failed to create rdma-core build directory"); + + println!("cargo:warning=Building rdma-core..."); + + // Detect cmake command + let cmake = if Command::new("cmake3").arg("--version").status().is_ok() { + "cmake3" + } else { + "cmake" + }; + + // Detect ninja + let use_ninja = Command::new("ninja-build") + .arg("--version") + .status() + .is_ok() + || Command::new("ninja").arg("--version").status().is_ok(); + + let ninja_cmd = if Command::new("ninja-build") + .arg("--version") + .status() + .is_ok() + { + "ninja-build" + } else { + "ninja" + }; + + // CMake configuration + // IMPORTANT: -DCMAKE_POSITION_INDEPENDENT_CODE=ON is required for static libs + // that will be linked into a shared object (.so) + let mut cmake_args = vec![ + "-DIN_PLACE=1", + "-DENABLE_STATIC=1", + "-DENABLE_RESOLVE_NEIGH=0", + "-DNO_PYVERBS=1", + "-DNO_MAN_PAGES=1", + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", + "-DCMAKE_C_FLAGS=-fPIC", + "-DCMAKE_CXX_FLAGS=-fPIC", + ]; + + if use_ninja { + cmake_args.push("-GNinja"); + } + + cmake_args.push(".."); + + let status = Command::new(cmake) + .current_dir(&build_dir) + .args(&cmake_args) + .status() + .expect("Failed to run cmake for rdma-core"); + + if !status.success() { + panic!("Failed to configure rdma-core with cmake"); + } + + // Build only the targets we need: libibverbs.a, libmlx5.a, and librdma_util.a + // We don't need librdmacm which has build issues with long paths + let targets = [ + "lib/statics/libibverbs.a", + "lib/statics/libmlx5.a", + "util/librdma_util.a", + ]; + + for target in &targets { + let status = if use_ninja { + Command::new(ninja_cmd) + .current_dir(&build_dir) + .arg(target) + .status() + .expect("Failed to run ninja for rdma-core") + } else { + let num_jobs = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(4); + Command::new("make") + .current_dir(&build_dir) + .args(["-j", &num_jobs.to_string(), target]) + .status() + .expect("Failed to run make for rdma-core") + }; + + if !status.success() { + panic!("Failed to build rdma-core target: {}", target); + } + } + + println!("cargo:warning=rdma-core build complete"); + build_dir +} + +fn build_nccl(nccl_dir: &Path, rdma_build_dir: &Path) -> PathBuf { + let build_dir = nccl_dir.join("build"); + + // Check if already built + if build_dir.join("lib/libnccl_static.a").exists() { + println!("cargo:warning=NCCL already built"); + return build_dir; + } + + println!("cargo:warning=Building NCCL..."); + + // Find CUDA + let cuda_home = build_utils::find_cuda_home().expect("CUDA not found"); + + // Set up environment + // IMPORTANT: -fPIC is required for static libs linked into shared objects + let rdma_include = rdma_build_dir.join("include"); + let cxxflags = format!("-fPIC -I{}", rdma_include.display()); + + let num_jobs = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(4); + + // Build NCCL with RDMA_CORE=1 and MLX5DV=1 + // Pass -Xcompiler -fPIC to nvcc for host code + let status = Command::new("make") + .current_dir(nccl_dir) + .env("CXXFLAGS", &cxxflags) + .env("CFLAGS", "-fPIC") + .args([ + &format!("-j{}", num_jobs), + "src.build", + &format!("CUDA_HOME={}", cuda_home), + "NVCC_GENCODE=-gencode=arch=compute_90,code=sm_90", + "RDMA_CORE=1", + "MLX5DV=1", + ]) + .status() + .expect("Failed to run make for NCCL"); + + if !status.success() { + panic!("Failed to build NCCL"); + } + + println!("cargo:warning=NCCL build complete"); + build_dir +} + +fn emit_link_directives(nccl_build_dir: &Path, rdma_build_dir: &Path) { + let nccl_lib_dir = nccl_build_dir.join("lib"); + let rdma_static_dir = rdma_build_dir.join("lib/statics"); + let rdma_util_dir = rdma_build_dir.join("util"); + + // Emit search paths + println!( + "cargo:rustc-link-search=native={}", + nccl_lib_dir.display() + ); + println!( + "cargo:rustc-link-search=native={}", + rdma_static_dir.display() + ); + println!( + "cargo:rustc-link-search=native={}", + rdma_util_dir.display() + ); + + // Static libraries - order matters for dependency resolution + // NCCL depends on rdma-core, so link NCCL first + + // Use whole-archive for NCCL static library + println!("cargo:rustc-link-arg=-Wl,--whole-archive"); + println!("cargo:rustc-link-lib=static=nccl_static"); + println!("cargo:rustc-link-arg=-Wl,--no-whole-archive"); + + // Use whole-archive for rdma-core static libraries + println!("cargo:rustc-link-arg=-Wl,--whole-archive"); + println!("cargo:rustc-link-lib=static=mlx5"); + println!("cargo:rustc-link-lib=static=ibverbs"); + println!("cargo:rustc-link-arg=-Wl,--no-whole-archive"); + + // rdma_util helper library + println!("cargo:rustc-link-lib=static=rdma_util"); + + // System libraries + println!("cargo:rustc-link-lib=cudart_static"); + println!("cargo:rustc-link-lib=pthread"); + println!("cargo:rustc-link-lib=rt"); + println!("cargo:rustc-link-lib=dl"); + + // Export metadata for dependent crates + // Use cargo:: (double colon) format for proper DEP__ env vars + println!( + "cargo::metadata=NCCL_INCLUDE={}", + nccl_build_dir.join("include").display() + ); + println!( + "cargo::metadata=RDMA_INCLUDE={}", + rdma_build_dir.join("include").display() + ); + println!("cargo::metadata=NCCL_LIB_DIR={}", nccl_lib_dir.display()); + println!( + "cargo::metadata=RDMA_LIB_DIR={}", + rdma_static_dir.display() + ); + println!( + "cargo::metadata=RDMA_UTIL_DIR={}", + rdma_util_dir.display() + ); + + // Re-run if build scripts change + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/monarch_cpp_static_libs/src/lib.rs b/monarch_cpp_static_libs/src/lib.rs new file mode 100644 index 000000000..d07ffffd9 --- /dev/null +++ b/monarch_cpp_static_libs/src/lib.rs @@ -0,0 +1,16 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Static C++ libraries for Monarch +//! +//! This crate builds NCCL and rdma-core (libibverbs, libmlx5) from source +//! as static libraries. Depend on this crate to link against them statically, +//! eliminating runtime dependencies on libnccl.so, libibverbs.so, and libmlx5.so. +//! +//! This crate does not provide Rust bindings - use `nccl-sys` and `rdmaxcel-sys` +//! for the bindings, and add this crate as a dependency to get static linking. diff --git a/monarch_extension/build.rs b/monarch_extension/build.rs index 432623496..d6659c3fa 100644 --- a/monarch_extension/build.rs +++ b/monarch_extension/build.rs @@ -7,11 +7,10 @@ */ fn main() { - // Only set torch-related rpaths if tensor_engine feature is enabled - #[cfg(feature = "tensor_engine")] - { - if let Ok(path) = std::env::var("DEP_NCCL_LIB_PATH") { - println!("cargo::rustc-link-arg=-Wl,-rpath,{path}"); - } + // Only set up static linking if tensor_engine feature is enabled + if std::env::var("CARGO_FEATURE_TENSOR_ENGINE").is_ok() { + // Set up static linking for NCCL and rdma-core + // This emits link directives for libnccl_static.a, libmlx5.a, libibverbs.a, librdma_util.a + let _config = build_utils::setup_nccl_static(); } } diff --git a/nccl-sys/Cargo.toml b/nccl-sys/Cargo.toml index 431130633..112923832 100644 --- a/nccl-sys/Cargo.toml +++ b/nccl-sys/Cargo.toml @@ -9,7 +9,11 @@ links = "nccl" [dependencies] cxx = "1.0.119" serde = { version = "1.0.185", features = ["derive", "rc"] } +# Depend on monarch_cpp_static_libs to get statically linked NCCL and rdma-core +monarch_cpp_static_libs = { path = "../monarch_cpp_static_libs" } [build-dependencies] bindgen = "0.70.1" build_utils = { path = "../build_utils" } +# Need monarch_cpp_static_libs as build-dep to get include paths via DEP_* env vars +monarch_cpp_static_libs = { path = "../monarch_cpp_static_libs" } diff --git a/nccl-sys/build.rs b/nccl-sys/build.rs index 9b4f7a97c..1bb333fed 100644 --- a/nccl-sys/build.rs +++ b/nccl-sys/build.rs @@ -13,6 +13,9 @@ fn main() {} #[cfg(not(target_os = "macos"))] fn main() { + // Get NCCL/rdma-core config from nccl-static-sys (includes are used, links emitted by monarch_extension) + let nccl_config = build_utils::NcclStaticConfig::from_env(); + let mut builder = bindgen::Builder::default() .header("src/nccl.h") .clang_arg("-x") @@ -22,6 +25,7 @@ fn main() { "-I{}/include", build_utils::find_cuda_home().unwrap() )) + .clang_arg(format!("-I{}", nccl_config.nccl_include)) .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) // Communicator creation and management .allowlist_function("ncclGetLastError") @@ -97,8 +101,6 @@ fn main() { } if let Some(lib_dir) = &python_config.lib_dir { println!("cargo::rustc-link-search=native={}", lib_dir); - // Set cargo metadata to inform dependent binaries about how to set their - // RPATH (see controller/build.rs for an example). println!("cargo::metadata=LIB_PATH={}", lib_dir); } @@ -110,7 +112,6 @@ fn main() { .write_to_file(out_path.join("bindings.rs")) .expect("Couldn't write bindings!"); - println!("cargo::rustc-link-lib=nccl"); println!("cargo::rustc-cfg=cargo"); println!("cargo::rustc-check-cfg=cfg(cargo)"); } diff --git a/rdmaxcel-sys/Cargo.toml b/rdmaxcel-sys/Cargo.toml index ee1d34f3b..dd448b145 100644 --- a/rdmaxcel-sys/Cargo.toml +++ b/rdmaxcel-sys/Cargo.toml @@ -9,8 +9,12 @@ links = "rdmaxcel" [dependencies] cxx = "1.0.119" serde = { version = "1.0.185", features = ["derive", "rc"] } +# Depend on monarch_cpp_static_libs to get statically linked rdma-core (libibverbs, libmlx5) +monarch_cpp_static_libs = { path = "../monarch_cpp_static_libs" } [build-dependencies] bindgen = "0.70.1" cc = "1.0" build_utils = { path = "../build_utils" } +# Need monarch_cpp_static_libs as build-dep to get include paths via DEP_* env vars +monarch_cpp_static_libs = { path = "../monarch_cpp_static_libs" } diff --git a/rdmaxcel-sys/build.rs b/rdmaxcel-sys/build.rs index 61ab2229d..5af267aa2 100644 --- a/rdmaxcel-sys/build.rs +++ b/rdmaxcel-sys/build.rs @@ -15,11 +15,9 @@ fn main() {} #[cfg(not(target_os = "macos"))] fn main() { - // Link against the ibverbs library - println!("cargo:rustc-link-lib=ibverbs"); - - // Link against the mlx5 library - println!("cargo:rustc-link-lib=mlx5"); + // Get NCCL/rdma-core config from nccl-static-sys (includes are used, links emitted by monarch_extension) + let nccl_config = build_utils::NcclStaticConfig::from_env(); + let rdma_include = &nccl_config.rdma_include; // Link against dl for dynamic loading println!("cargo:rustc-link-lib=dl"); @@ -140,6 +138,9 @@ fn main() { println!("cargo:rustc-env=CUDA_INCLUDE_PATH={}", cuda_include_path); builder = builder.clang_arg(format!("-I{}", cuda_include_path)); + // Add rdma-core include path from nccl-static-sys + builder = builder.clang_arg(format!("-I{}", rdma_include)); + // Include headers and libs from the active environment. let python_config = match build_utils::python_env_dirs_with_interpreter("python3") { Ok(config) => config, @@ -170,8 +171,12 @@ fn main() { }; println!("cargo:rustc-link-search=native={}", cuda_lib_dir); // Note: libcuda is now loaded dynamically via dlopen in driver_api.cpp - // Only link cudart (CUDA Runtime API) - println!("cargo:rustc-link-lib=cudart"); + // Link cudart statically (CUDA Runtime API) + println!("cargo:rustc-link-lib=static=cudart_static"); + // cudart_static requires linking against librt and libpthread + println!("cargo:rustc-link-lib=rt"); + println!("cargo:rustc-link-lib=pthread"); + println!("cargo:rustc-link-lib=dl"); // Note: We no longer link against libtorch/c10 since segment scanning // is now done via a callback registered from the extension crate. @@ -201,6 +206,7 @@ fn main() { build .file(&c_source_path) .include(format!("{}/src", manifest_dir)) + .include(&rdma_include) .flag("-fPIC"); // Add CUDA include paths - reuse the paths we already found for bindgen @@ -220,6 +226,7 @@ fn main() { .file(&cpp_source_path) .file(&driver_api_cpp_path) .include(format!("{}/src", manifest_dir)) + .include(&rdma_include) .flag("-fPIC") .cpp(true) .flag("-std=c++14"); @@ -233,6 +240,9 @@ fn main() { } cpp_build.compile("rdmaxcel_cpp"); + + // Statically link libstdc++ to avoid runtime dependency on system libstdc++ + build_utils::link_libstdcpp_static(); } else { if !Path::new(&cpp_source_path).exists() { panic!("C++ source file not found at {}", cpp_source_path); @@ -273,8 +283,7 @@ fn main() { "-fPIC", &format!("-I{}", cuda_include_path), &format!("-I{}/src", manifest_dir), - &format!("-I/usr/include"), - &format!("-I/usr/include/infiniband"), + &format!("-I{}", rdma_include), ]) .output(); diff --git a/torch-sys-cuda/build.rs b/torch-sys-cuda/build.rs index bf6928a74..f4c7e577f 100644 --- a/torch-sys-cuda/build.rs +++ b/torch-sys-cuda/build.rs @@ -52,8 +52,15 @@ fn main() { .flag_if_supported("-w") .compile("torch-sys-cuda"); - // Configure CUDA-specific linking - println!("cargo::rustc-link-lib=cudart"); + // Statically link libstdc++ to avoid runtime dependency on system libstdc++ + build_utils::link_libstdcpp_static(); + + // Configure CUDA-specific linking - use static cudart + println!("cargo::rustc-link-lib=static=cudart_static"); + // cudart_static requires linking against librt and libpthread + println!("cargo::rustc-link-lib=rt"); + println!("cargo::rustc-link-lib=pthread"); + println!("cargo::rustc-link-lib=dl"); println!( "cargo::rustc-link-search=native={}/lib64", cuda_home.display()