Link statically

Pull Request resolved: #2085

Link everything into monarch statically, or dlopen it.

This makes the rdma related libraries and libc++ linked statically for increased portability. This means monarch built with rdma support and with a different version of libc++ than the current system can still be used.

By building rdma-core from source, we can also make sure Dennis' 64-bit patch is applied.

Total size of our so is 63MB now with no uncommon deps:

```
(monarch) [zdevito@devgpu014 /data/users/zdevito/fbsource/fbcode/monarch]$ ldd ./python/monarch/_rust_bindings.so
        linux-vdso.so.1 (0x00007f347a684000)
        libpython3.11.so.1.0 => /home/zdevito/local/miniconda3/envs/monarch/lib/libpython3.11.so.1.0 (0x00007f3477200000)
        libgcc_s.so.1 => /home/zdevito/local/miniconda3/envs/monarch/lib/libgcc_s.so.1 (0x00007f347a664000)
        libm.so.6 => /lib64/libm.so.6 (0x00007f3477125000)
        libc.so.6 => /lib64/libc.so.6 (0x00007f3476e00000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f347a686000)
        libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f347a64a000)
        libdl.so.2 => /lib64/libdl.so.2 (0x00007f347a645000)
        libutil.so.1 => /lib64/libutil.so.1 (0x00007f347a640000)
(monarch) [zdevito@devgpu014 /data/users/zdevito/fbsource/fbcode/monarch]$ du -h ./python/monarch/_rust_bindings.so
63M     ./python/monarch/_rust_bindings.so
```
ghstack-source-id: 328253183

Differential Revision: [D88540873](https://our.internmc.facebook.com/intern/diff/D88540873/)

Author: zdevito

.gitignore

@@ -33,4 +33,3 @@ docs/_build/**
 docs/build/**
 docs/**/generated/**
 */sg_execution_times.rst
-nccl/**

Cargo.toml

@@ -17,6 +17,7 @@ members = [
     "monarch_rdma",
     "monarch_tensor_worker",
     "monarch_types",
+    "monarch_cpp_static_libs",
     "nccl-sys",
     "ndslice",
     "preempt_rwlock",

build_utils/src/lib.rs

@@ -276,6 +276,100 @@ pub fn print_cuda_lib_error_help() {
     eprintln!("Or: export CUDA_LIB_DIR=/usr/lib64");
 }
 
+/// Emit cargo directives to statically link libstdc++
+///
+/// This finds the GCC library path containing libstdc++.a and emits the
+/// appropriate cargo directives to link it statically. This avoids runtime
+/// dependency on system libstdc++.so which can cause GLIBCXX version conflicts.
+///
+/// Uses the `cc` crate to detect the C++ compiler, ensuring we use the same
+/// compiler that `cc::Build` and `cxx_build` would use.
+pub fn link_libstdcpp_static() {
+    // Use cc crate to get the C++ compiler, same as cc::Build and cxx_build use
+    let compiler = cc::Build::new().cpp(true).get_compiler();
+    let gcc_lib_path = std::process::Command::new(compiler.path())
+        .args(["-print-file-name=libstdc++.a"])
+        .output()
+        .ok()
+        .and_then(|output| {
+            if output.status.success() {
+                String::from_utf8(output.stdout).ok().and_then(|s| {
+                    let path = PathBuf::from(s.trim());
+                    path.parent().map(|p| p.to_path_buf())
+                })
+            } else {
+                None
+            }
+        });
+    if let Some(gcc_lib_path) = gcc_lib_path {
+        println!("cargo:rustc-link-search=native={}", gcc_lib_path.display());
+    }
+    println!("cargo:rustc-link-lib=static=stdc++");
+}
+
+/// Configuration for rdma-core static libraries from monarch_cpp_static_libs.
+///
+/// Use `CppStaticLibsConfig::from_env()` to get the paths, then use the include
+/// paths for bindgen/cc, and call `emit_link_directives()` to link.
+pub struct CppStaticLibsConfig {
+    pub rdma_include: String,
+    pub rdma_lib_dir: String,
+    pub rdma_util_dir: String,
+}
+
+impl CppStaticLibsConfig {
+    /// Load configuration from DEP_* environment variables set by monarch_cpp_static_libs.
+    ///
+    /// The monarch_cpp_static_libs crate must be listed as a build-dependency.
+    pub fn from_env() -> Self {
+        Self {
+            rdma_include: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE")
+                .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE not set - add monarch_cpp_static_libs as build-dependency"),
+            rdma_lib_dir: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_LIB_DIR")
+                .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_LIB_DIR not set - add monarch_cpp_static_libs as build-dependency"),
+            rdma_util_dir: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_UTIL_DIR")
+                .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_UTIL_DIR not set - add monarch_cpp_static_libs as build-dependency"),
+        }
+    }
+
+    /// Emit all cargo link directives for static linking of rdma-core.
+    ///
+    /// This emits search paths and link-lib directives for:
+    /// - libmlx5.a
+    /// - libibverbs.a
+    /// - librdma_util.a
+    pub fn emit_link_directives(&self) {
+        // Emit link search paths
+        println!("cargo::rustc-link-search=native={}", self.rdma_lib_dir);
+        println!("cargo::rustc-link-search=native={}", self.rdma_util_dir);
+
+        // Use whole-archive for rdma-core static libraries
+        println!("cargo::rustc-link-arg=-Wl,--whole-archive");
+        println!("cargo::rustc-link-lib=static=mlx5");
+        println!("cargo::rustc-link-lib=static=ibverbs");
+        println!("cargo::rustc-link-arg=-Wl,--no-whole-archive");
+
+        // rdma_util helper library
+        println!("cargo::rustc-link-lib=static=rdma_util");
+    }
+}
+
+/// Convenience function to set up rdma-core static linking.
+///
+/// Returns the config with include paths, and emits all link directives.
+/// The monarch_cpp_static_libs crate must be listed as a build-dependency.
+///
+/// Example:
+/// ```ignore
+/// let config = build_utils::setup_cpp_static_libs();
+/// // Use config.rdma_include for bindgen/cc
+/// ```
+pub fn setup_cpp_static_libs() -> CppStaticLibsConfig {
+    let config = CppStaticLibsConfig::from_env();
+    config.emit_link_directives();
+    config
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

monarch_cpp_static_libs/build.rs

@@ -0,0 +1,277 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//! Static rdma-core build script
+//!
+//! This build script:
+//! 1. Obtains rdma-core source (from MONARCH_RDMA_CORE_SRC or by cloning)
+//! 2. Builds rdma-core with static libraries (libibverbs.a, libmlx5.a)
+//! 3. Emits link directives for downstream crates
+
+use std::path::Path;
+use std::path::PathBuf;
+use std::process::Command;
+
+// Repository configuration
+const RDMA_CORE_REPO: &str = "https://github.com/linux-rdma/rdma-core";
+const RDMA_CORE_TAG: &str = "224154663a9ad5b1ad5629fb76a0c40c675fb936";
+
+#[cfg(target_os = "macos")]
+fn main() {}
+
+#[cfg(not(target_os = "macos"))]
+fn main() {
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set"));
+    let vendor_dir = out_dir.join("vendor");
+    std::fs::create_dir_all(&vendor_dir).expect("Failed to create vendor directory");
+
+    let rdma_core_dir = vendor_dir.join("rdma-core");
+
+    // Get or clone rdma-core source
+    get_or_clone_rdma_core(&rdma_core_dir);
+
+    // Build rdma-core
+    let rdma_build_dir = build_rdma_core(&rdma_core_dir);
+
+    // Emit link directives
+    emit_link_directives(&rdma_build_dir);
+}
+
+/// Get or clone rdma-core source.
+///
+/// If MONARCH_RDMA_CORE_SRC is set, copies from that directory.
+/// Otherwise, clones from GitHub at the specified tag.
+fn get_or_clone_rdma_core(target_dir: &Path) {
+    // Skip if already exists
+    if target_dir.exists() {
+        println!(
+            "cargo:warning=rdma-core source already exists at {}",
+            target_dir.display()
+        );
+        return;
+    }
+
+    // Check for MONARCH_RDMA_CORE_SRC environment variable
+    println!("cargo:rerun-if-env-changed=MONARCH_RDMA_CORE_SRC");
+    if let Ok(src_path) = std::env::var("MONARCH_RDMA_CORE_SRC") {
+        let src_dir = PathBuf::from(src_path);
+        println!(
+            "cargo:warning=Using rdma-core source from MONARCH_RDMA_CORE_SRC: {}",
+            src_dir.display()
+        );
+        copy_dir(&src_dir, target_dir);
+    } else {
+        println!(
+            "cargo:warning=MONARCH_RDMA_CORE_SRC not set, cloning from {} (commit {})",
+            RDMA_CORE_REPO, RDMA_CORE_TAG
+        );
+        clone_rdma_core(target_dir);
+    }
+}
+
+/// Clone rdma-core from GitHub at the specified commit.
+fn clone_rdma_core(target_dir: &Path) {
+    // First, clone the repository without checking out
+    let status = Command::new("git")
+        .args([
+            "clone",
+            "--no-checkout",
+            RDMA_CORE_REPO,
+            target_dir.to_str().unwrap(),
+        ])
+        .status()
+        .expect("Failed to execute git clone");
+
+    if !status.success() {
+        panic!("Failed to clone rdma-core from {}", RDMA_CORE_REPO);
+    }
+
+    // Then checkout the specific commit
+    let status = Command::new("git")
+        .args(["checkout", RDMA_CORE_TAG])
+        .current_dir(target_dir)
+        .status()
+        .expect("Failed to execute git checkout");
+
+    if !status.success() {
+        panic!("Failed to checkout rdma-core commit {}", RDMA_CORE_TAG);
+    }
+
+    println!(
+        "cargo:warning=Successfully cloned rdma-core at commit {}",
+        RDMA_CORE_TAG
+    );
+}
+
+fn copy_dir(src_dir: &Path, target_dir: &Path) {
+    if target_dir.exists() {
+        println!(
+            "cargo:warning=Directory already exists at {}",
+            target_dir.display()
+        );
+        return;
+    }
+
+    println!(
+        "cargo:warning=Copying {} to {}",
+        src_dir.display(),
+        target_dir.display()
+    );
+
+    let status = Command::new("cp")
+        .args([
+            "-r",
+            src_dir.to_str().unwrap(),
+            target_dir.to_str().unwrap(),
+        ])
+        .status()
+        .expect("Failed to execute cp");
+
+    if !status.success() {
+        panic!(
+            "Failed to copy from {} to {}",
+            src_dir.display(),
+            target_dir.display()
+        );
+    }
+}
+
+fn build_rdma_core(rdma_core_dir: &Path) -> PathBuf {
+    let build_dir = rdma_core_dir.join("build");
+
+    // Check if already built
+    if build_dir.join("lib/statics/libibverbs.a").exists() {
+        println!("cargo:warning=rdma-core already built");
+        return build_dir;
+    }
+
+    std::fs::create_dir_all(&build_dir).expect("Failed to create rdma-core build directory");
+
+    println!("cargo:warning=Building rdma-core...");
+
+    // Detect cmake command
+    let cmake = if Command::new("cmake3").arg("--version").status().is_ok() {
+        "cmake3"
+    } else {
+        "cmake"
+    };
+
+    // Detect ninja
+    let use_ninja = Command::new("ninja-build")
+        .arg("--version")
+        .status()
+        .is_ok()
+        || Command::new("ninja").arg("--version").status().is_ok();
+
+    let ninja_cmd = if Command::new("ninja-build")
+        .arg("--version")
+        .status()
+        .is_ok()
+    {
+        "ninja-build"
+    } else {
+        "ninja"
+    };
+
+    // CMake configuration
+    // IMPORTANT: -DCMAKE_POSITION_INDEPENDENT_CODE=ON is required for static libs
+    // that will be linked into a shared object (.so)
+    let mut cmake_args = vec![
+        "-DIN_PLACE=1",
+        "-DENABLE_STATIC=1",
+        "-DENABLE_RESOLVE_NEIGH=0",
+        "-DNO_PYVERBS=1",
+        "-DNO_MAN_PAGES=1",
+        "-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
+        "-DCMAKE_C_FLAGS=-fPIC",
+        "-DCMAKE_CXX_FLAGS=-fPIC",
+    ];
+
+    if use_ninja {
+        cmake_args.push("-GNinja");
+    }
+
+    cmake_args.push("..");
+
+    let status = Command::new(cmake)
+        .current_dir(&build_dir)
+        .args(&cmake_args)
+        .status()
+        .expect("Failed to run cmake for rdma-core");
+
+    if !status.success() {
+        panic!("Failed to configure rdma-core with cmake");
+    }
+
+    // Build only the targets we need: libibverbs.a, libmlx5.a, and librdma_util.a
+    // We don't need librdmacm which has build issues with long paths
+    let targets = [
+        "lib/statics/libibverbs.a",
+        "lib/statics/libmlx5.a",
+        "util/librdma_util.a",
+    ];
+
+    for target in &targets {
+        let status = if use_ninja {
+            Command::new(ninja_cmd)
+                .current_dir(&build_dir)
+                .arg(target)
+                .status()
+                .expect("Failed to run ninja for rdma-core")
+        } else {
+            let num_jobs = std::thread::available_parallelism()
+                .map(|p| p.get())
+                .unwrap_or(4);
+            Command::new("make")
+                .current_dir(&build_dir)
+                .args(["-j", &num_jobs.to_string(), target])
+                .status()
+                .expect("Failed to run make for rdma-core")
+        };
+
+        if !status.success() {
+            panic!("Failed to build rdma-core target: {}", target);
+        }
+    }
+
+    println!("cargo:warning=rdma-core build complete");
+    build_dir
+}
+
+fn emit_link_directives(rdma_build_dir: &Path) {
+    let rdma_static_dir = rdma_build_dir.join("lib/statics");
+    let rdma_util_dir = rdma_build_dir.join("util");
+
+    // Emit search paths
+    println!(
+        "cargo:rustc-link-search=native={}",
+        rdma_static_dir.display()
+    );
+    println!("cargo:rustc-link-search=native={}", rdma_util_dir.display());
+
+    // Static libraries - use whole-archive for rdma-core static libraries
+    println!("cargo:rustc-link-arg=-Wl,--whole-archive");
+    println!("cargo:rustc-link-lib=static=mlx5");
+    println!("cargo:rustc-link-lib=static=ibverbs");
+    println!("cargo:rustc-link-arg=-Wl,--no-whole-archive");
+
+    // rdma_util helper library
+    println!("cargo:rustc-link-lib=static=rdma_util");
+
+    // Export metadata for dependent crates
+    // Use cargo:: (double colon) format for proper DEP_<LINKS>_<KEY> env vars
+    println!(
+        "cargo::metadata=RDMA_INCLUDE={}",
+        rdma_build_dir.join("include").display()
+    );
+    println!("cargo::metadata=RDMA_LIB_DIR={}", rdma_static_dir.display());
+    println!("cargo::metadata=RDMA_UTIL_DIR={}", rdma_util_dir.display());
+
+    // Re-run if build scripts change
+    println!("cargo:rerun-if-changed=build.rs");
+}