From 4006ef56c5cd6e5f6a899749eacd9bf67fc75da1 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 16:12:49 -0500 Subject: [PATCH 01/48] update offload build steps --- src/bootstrap/src/core/build_steps/llvm.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index a591be05291fa..8776ffd751fdd 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -462,7 +462,13 @@ impl Step for Llvm { //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. enabled_llvm_runtimes.push("openmp"); - enabled_llvm_projects.push("compiler-rt"); + enabled_llvm_runtimes.push("compiler-rt"); + + let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + + cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); + cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } if !enabled_llvm_projects.is_empty() { From 710bcab75e8d6c638b17b44e89cc907782b3bda4 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 16:23:57 -0500 Subject: [PATCH 02/48] I surely picked the wrong runner again, right? --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index cb57478761994..ba62750643d4c 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -90,6 +90,7 @@ ENV RUST_CONFIGURE_ARGS \ --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ + --set llvm.offload=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 1ea13fcb03ee0e20eb59b513ad9c90869771c382 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 18:04:49 -0500 Subject: [PATCH 03/48] try fix --- src/bootstrap/src/core/build_steps/llvm.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 8776ffd751fdd..14a1f19bf5a85 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -464,11 +464,15 @@ impl Step for Llvm { enabled_llvm_runtimes.push("openmp"); enabled_llvm_runtimes.push("compiler-rt"); - let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; - cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; + cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); + //cfg.define("LLVM_TARGETS_TO_BUILD", "host;AMDGPU;NVPTX"); - cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); - cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); + //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + + //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); + //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } if !enabled_llvm_projects.is_empty() { From 10aec1a47ff0d815711d95f4618f1d885349d079 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 21:53:46 -0500 Subject: [PATCH 04/48] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index ba62750643d4c..6e93d29f03605 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -91,6 +91,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set llvm.offload=true \ + --set llvm.clang=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From e8d3c616919b4087ba44b5e38e512141cc99e352 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 14 Nov 2025 18:45:51 -0500 Subject: [PATCH 05/48] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 6e93d29f03605..2a80940ad1056 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -92,6 +92,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set llvm.offload=true \ --set llvm.clang=true \ + --set llvm.lld=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 23b7e63f20524b833ca63e4e319419a4abf2b7bf Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 15 Nov 2025 17:50:27 -0500 Subject: [PATCH 06/48] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 2a80940ad1056..8800c2fbe7053 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -92,7 +92,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set llvm.offload=true \ --set llvm.clang=true \ - --set llvm.lld=true \ + --set rust.lld=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 9af17182de4879168f6ef07ae1aa356fc1dfa6eb Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 15 Nov 2025 21:03:12 -0500 Subject: [PATCH 07/48] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 8800c2fbe7053..355c387550445 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -93,6 +93,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.offload=true \ --set llvm.clang=true \ --set rust.lld=true \ + --set llvm.cxxflags = "-fuse-ld=lld" \ + --set llvm.cflags = "-fuse-ld=lld" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 8f960bc6a6a543e0ae425df144a60e812b08b99f Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sun, 16 Nov 2025 03:20:09 -0500 Subject: [PATCH 08/48] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 355c387550445..d02b0eaf260bd 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -93,8 +93,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.offload=true \ --set llvm.clang=true \ --set rust.lld=true \ - --set llvm.cxxflags = "-fuse-ld=lld" \ - --set llvm.cflags = "-fuse-ld=lld" \ + --set llvm.cxxflags="-fuse-ld=lld" \ + --set llvm.cflags="-fuse-ld=lld" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 0851b59aa86d86fc02abd379cff1febf207c142a Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sun, 16 Nov 2025 09:33:38 -0500 Subject: [PATCH 09/48] try fix --- src/bootstrap/src/core/build_steps/llvm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 14a1f19bf5a85..3e83eeeb0d337 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -458,6 +458,7 @@ impl Step for Llvm { // This is an experimental flag, which likely builds more than necessary. // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { + enabled_llvm_projects.push("lld"); enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. From 825d0fcd92aafc9cd72140ef639a786668c9461d Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sun, 16 Nov 2025 19:23:38 -0500 Subject: [PATCH 10/48] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index d02b0eaf260bd..ba62750643d4c 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -91,10 +91,6 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set llvm.offload=true \ - --set llvm.clang=true \ - --set rust.lld=true \ - --set llvm.cxxflags="-fuse-ld=lld" \ - --set llvm.cflags="-fuse-ld=lld" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From b73bdf193be1c2ad6f0ec598a5ac82dc24b10eaa Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 14:28:05 -0500 Subject: [PATCH 11/48] disable compiler-rt --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 3e83eeeb0d337..eac234b5b416c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -463,7 +463,7 @@ impl Step for Llvm { //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. enabled_llvm_runtimes.push("openmp"); - enabled_llvm_runtimes.push("compiler-rt"); + //enabled_llvm_runtimes.push("compiler-rt"); // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); From d6521a3785b9ca17a8d5198c1bdf43ce0420f8a3 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 14:31:14 -0500 Subject: [PATCH 12/48] disable lld project --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index eac234b5b416c..7f2b99a5a8965 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -458,7 +458,7 @@ impl Step for Llvm { // This is an experimental flag, which likely builds more than necessary. // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { - enabled_llvm_projects.push("lld"); + //enabled_llvm_projects.push("lld"); enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. From a45154807d22182486946d7ba58a7c318f1bfabb Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 14:51:37 -0500 Subject: [PATCH 13/48] disable test building --- src/bootstrap/src/core/build_steps/llvm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 7f2b99a5a8965..f027dd000f9d7 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -472,6 +472,7 @@ impl Step for Llvm { //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + cfg.define("-DLLVM_BUILD_TESTS", "OFF"); //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } From f366f03bb7fa5735d880f3ec753fc527e9d6e374 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 15:37:10 -0500 Subject: [PATCH 14/48] try again to disable tests --- src/bootstrap/src/core/build_steps/llvm.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index f027dd000f9d7..34f3948ceb195 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -472,7 +472,8 @@ impl Step for Llvm { //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); - cfg.define("-DLLVM_BUILD_TESTS", "OFF"); + cfg.define("LLVM_INCLUDE_TESTS", "OFF"); + cfg.define("LLVM_BUILD_TESTS", "OFF"); //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } From 418465e307dceeffeb0aafd43dd8f97120a4c2ad Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 16:32:15 -0500 Subject: [PATCH 15/48] temporarily test if we can at least build offload without openmp --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 34f3948ceb195..eb6cd9a9cb94b 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -462,7 +462,7 @@ impl Step for Llvm { enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. - enabled_llvm_runtimes.push("openmp"); + //enabled_llvm_runtimes.push("openmp"); //enabled_llvm_runtimes.push("compiler-rt"); // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; From 19e19384ab7d17dc441e52ac9bdd1d078b7f5cb1 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 16:52:23 -0500 Subject: [PATCH 16/48] disable RUNTIMES_CMAKE_ARGS --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index eb6cd9a9cb94b..7cc2fd3d29448 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -466,7 +466,7 @@ impl Step for Llvm { //enabled_llvm_runtimes.push("compiler-rt"); // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; - cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); + //cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); //cfg.define("LLVM_TARGETS_TO_BUILD", "host;AMDGPU;NVPTX"); //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; From cc81414beea77b91f2a44bc053f550d86ba34f31 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 27 Nov 2025 12:54:01 -0800 Subject: [PATCH 17/48] working standalone openmp build --- src/bootstrap/src/core/build_steps/compile.rs | 5 + src/bootstrap/src/core/build_steps/llvm.rs | 129 +++++++++++++++++- 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 02940a80295b7..49a50f1af72c6 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1444,6 +1444,11 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_OFFLOAD", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); + if builder.config.llvm_enzyme { + builder.ensure(llvm::Offload { target }); + cargo.env("LLVM_OFFLOAD", "1"); + } + cargo.env("LLVM_CONFIG", &host_llvm_config); // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 7cc2fd3d29448..479e2f5ba8514 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -459,7 +459,7 @@ impl Step for Llvm { // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { //enabled_llvm_projects.push("lld"); - enabled_llvm_runtimes.push("offload"); + //enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. //enabled_llvm_runtimes.push("openmp"); @@ -472,8 +472,8 @@ impl Step for Llvm { //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); - cfg.define("LLVM_INCLUDE_TESTS", "OFF"); - cfg.define("LLVM_BUILD_TESTS", "OFF"); + //cfg.define("LLVM_INCLUDE_TESTS", "OFF"); + //cfg.define("LLVM_BUILD_TESTS", "OFF"); //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } @@ -910,6 +910,129 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { .or_else(|| env::var_os(var_base)) } +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Offload { + pub target: TargetSelection, +} + +impl Step for Offload { + type Output = PathBuf; + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/openmp") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Offload { target: run.target }); + } + + /// Compile OpenMP offload runtimes for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::from("/"); + } + let target = self.target; + + let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + + let out_dir = builder.llvm_out(target); + static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); + //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + // generate_smart_stamp_hash( + // builder, + // &builder.config.src.join("src/llvm-project/openmp"), + // builder.offload_info.sha().unwrap_or_default(), + // ) + //}); + //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); + + trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); + //if stamp.is_up_to_date() { + // trace!(?out_dir, "offload/openmp build artifacts are up to date"); + // if stamp.stamp().is_empty() { + // builder.info( + // "Could not determine the Offload submodule commit hash. \ + // Assuming that an Offload rebuild is not necessary.", + // ); + // builder.info(&format!( + // "To force Offload to rebuild, remove the file `{}`", + // stamp.path().display() + // )); + // } + // return out_dir; + //} + + trace!(?target, "(re)building openmp artifacts"); + builder.info(&format!("Building OpenMP for {target}")); + //t!(stamp.remove()); + let _time = helpers::timeit(builder); + t!(fs::create_dir_all(&out_dir)); + + builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); + configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); + + // Re-use the same flags as llvm to control the level of debug information + // generated by Enzyme. + // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + trace!(?profile); + + cfg.out_dir(&out_dir) + .profile(profile) + .env("LLVM_CONFIG_REAL", &host_llvm_config) + .define("LLVM_ENABLE_ASSERTIONS", "ON") + .define("TARGET_TRIPLE", &target.triple) + .define("LLVM_ENABLE_RUNTIMES", "openmp") + .define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + .define("OPENMP_STANDALONE_BUILD", "ON") + .define( + "LLVM_ROOT", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", + ) + .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ + // -DCMAKE_BUILD_TYPE=Release + cfg.build(); + + //t!(stamp.write()); + out_dir + + //$> cd llvm-project # The llvm-project checkout + //$> mkdir build # A different build directory for the build tools + //$> cd build + //$> TARGET_TRIPLE= + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> ninja install + // + //You can do the same thing for the offload project. + // + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DCMAKE_BUILD_TYPE=Release + //$> ninja install + } +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Enzyme { pub target: TargetSelection, From c7efa423ccbbd62487e179593ca3a9c398beefac Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 27 Nov 2025 13:33:43 -0800 Subject: [PATCH 18/48] working standalone offload build, fails runtime testing --- src/bootstrap/src/core/build_steps/compile.rs | 1 + src/bootstrap/src/core/build_steps/llvm.rs | 123 ++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 49a50f1af72c6..fe048b97e32c2 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1445,6 +1445,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_enzyme { + builder.ensure(llvm::Openmp { target }); builder.ensure(llvm::Offload { target }); cargo.env("LLVM_OFFLOAD", "1"); } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 479e2f5ba8514..3764a2a41f5c7 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -910,6 +910,129 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { .or_else(|| env::var_os(var_base)) } +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Openmp { + pub target: TargetSelection, +} + +impl Step for Openmp { + type Output = PathBuf; + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/openmp") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Openmp { target: run.target }); + } + + /// Compile OpenMP offload runtimes for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::from("/"); + } + let target = self.target; + + let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + + let out_dir = builder.llvm_out(target); + static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); + //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + // generate_smart_stamp_hash( + // builder, + // &builder.config.src.join("src/llvm-project/openmp"), + // builder.offload_info.sha().unwrap_or_default(), + // ) + //}); + //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); + + trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); + //if stamp.is_up_to_date() { + // trace!(?out_dir, "offload/openmp build artifacts are up to date"); + // if stamp.stamp().is_empty() { + // builder.info( + // "Could not determine the Offload submodule commit hash. \ + // Assuming that an Offload rebuild is not necessary.", + // ); + // builder.info(&format!( + // "To force Offload to rebuild, remove the file `{}`", + // stamp.path().display() + // )); + // } + // return out_dir; + //} + + trace!(?target, "(re)building openmp artifacts"); + builder.info(&format!("Building OpenMP for {target}")); + //t!(stamp.remove()); + let _time = helpers::timeit(builder); + t!(fs::create_dir_all(&out_dir)); + + builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); + configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); + + // Re-use the same flags as llvm to control the level of debug information + // generated by Enzyme. + // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + trace!(?profile); + + cfg.out_dir(&out_dir) + .profile(profile) + .env("LLVM_CONFIG_REAL", &host_llvm_config) + .define("LLVM_ENABLE_ASSERTIONS", "ON") + .define("TARGET_TRIPLE", &target.triple) + .define("LLVM_ENABLE_RUNTIMES", "offload") + .define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + .define("OFFLOAD_STANDALONE_BUILD", "ON") + .define( + "LLVM_ROOT", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", + ) + .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ + // -DCMAKE_BUILD_TYPE=Release + cfg.build(); + + //t!(stamp.write()); + out_dir + + //$> cd llvm-project # The llvm-project checkout + //$> mkdir build # A different build directory for the build tools + //$> cd build + //$> TARGET_TRIPLE= + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> ninja install + // + //You can do the same thing for the offload project. + // + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DCMAKE_BUILD_TYPE=Release + //$> ninja install + } +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Offload { pub target: TargetSelection, From d5f904a637919e271020b8a330b10c54a9f09878 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 27 Nov 2025 14:18:58 -0800 Subject: [PATCH 19/48] wip --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- src/bootstrap/src/core/build_steps/llvm.rs | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index fe048b97e32c2..4920292bb6b2c 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1444,7 +1444,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_OFFLOAD", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_enzyme { + if builder.config.llvm_offload { builder.ensure(llvm::Openmp { target }); builder.ensure(llvm::Offload { target }); cargo.env("LLVM_OFFLOAD", "1"); diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 3764a2a41f5c7..d83747eb5871c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -989,8 +989,16 @@ impl Step for Openmp { .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("TARGET_TRIPLE", &target.triple) .define("LLVM_ENABLE_RUNTIMES", "offload") - .define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + //.define("CMAKE_C_COMPILER", builder.cc(target)) + //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define( + "CMAKE_C_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", + ) + .define( + "CMAKE_CXX_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", + ) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OFFLOAD_STANDALONE_BUILD", "ON") .define( @@ -1043,7 +1051,7 @@ impl Step for Offload { const IS_HOST: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/openmp") + run.path("src/llvm-project/offload") } fn make_run(run: RunConfig<'_>) { @@ -1064,7 +1072,7 @@ impl Step for Offload { //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { // generate_smart_stamp_hash( // builder, - // &builder.config.src.join("src/llvm-project/openmp"), + // &builder.config.src.join("src/llvm-project/offload"), // builder.offload_info.sha().unwrap_or_default(), // ) //}); @@ -1086,8 +1094,8 @@ impl Step for Offload { // return out_dir; //} - trace!(?target, "(re)building openmp artifacts"); - builder.info(&format!("Building OpenMP for {target}")); + trace!(?target, "(re)building offload artifacts"); + builder.info(&format!("Building Offload for {target}")); //t!(stamp.remove()); let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); @@ -1112,8 +1120,16 @@ impl Step for Offload { .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("TARGET_TRIPLE", &target.triple) .define("LLVM_ENABLE_RUNTIMES", "openmp") - .define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define( + "CMAKE_C_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", + ) + .define( + "CMAKE_CXX_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", + ) + //.define("CMAKE_C_COMPILER", builder.cc(target)) + //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OPENMP_STANDALONE_BUILD", "ON") .define( From 46c70355987ebbef4228b0178f10cb42499e5e25 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 1 Dec 2025 19:14:35 -0800 Subject: [PATCH 20/48] try to not hardcode paths --- src/bootstrap/src/core/build_steps/compile.rs | 3 -- src/bootstrap/src/core/build_steps/llvm.rs | 47 ++++++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 4920292bb6b2c..4b862e2eea336 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1440,9 +1440,6 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect if builder.config.llvm_enzyme { cargo.env("LLVM_ENZYME", "1"); } - if builder.config.llvm_offload { - cargo.env("LLVM_OFFLOAD", "1"); - } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_offload { builder.ensure(llvm::Openmp { target }); diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d83747eb5871c..8673e3d7e017f 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -991,22 +991,43 @@ impl Step for Openmp { .define("LLVM_ENABLE_RUNTIMES", "offload") //.define("CMAKE_C_COMPILER", builder.cc(target)) //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - .define( - "CMAKE_C_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", - ) - .define( - "CMAKE_CXX_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", - ) + //.define( + // "CMAKE_C_COMPILER", + // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", + //) + //.define( + // "CMAKE_CXX_COMPILER", + // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", + //) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OFFLOAD_STANDALONE_BUILD", "ON") - .define( - "LLVM_ROOT", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", - ) + .define("LLVM_ROOT", builder.llvm_out(target)) + //.define( + // "LLVM_ROOT", + // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", + //) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm + + if builder.config.llvm_clang { + // This is likely the case locally. If we just build clang in the previous step, we + // should use it. We wouldn't even need a standalone build here, but it's easier to + // unify this with the case below, which needs standalone builds. + // We likely wouldn't need to specify these here, they should be found by default. + cfg.define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); + } else { + // This is the (more complicated) case which we have e.g. in CI. We first build a + // standalone llvm/llvm-project including clang. Then we use this clang to build the + // llvm submodule of rustc in src/llvm-project. In this case we do *NOT* build clang + // again in the llvm submodule to save compile times, this is a hard requirement. + // If we build the offload and openmp runtimes in-tree, they assume that clang was + // already build in-tree, which is not the case here. We therefore must use standalone + // builds of these two runtimes. We now have to specify clang(++) manually to match the + // one we build in llvm/llvm-project + cfg.define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); + } + // /tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm //$> cmake ../runtimes \ # Point to the runtimes build // -G Ninja \ // -DLLVM_ENABLE_RUNTIMES=openmp \ From 5ae5def432434369b8371ed4c7dfafb43ad4fce2 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 1 Dec 2025 19:46:15 -0800 Subject: [PATCH 21/48] unify both ol and omp runtime builds into one step --- src/bootstrap/src/core/build_steps/compile.rs | 3 +- src/bootstrap/src/core/build_steps/llvm.rs | 205 ++---------------- 2 files changed, 14 insertions(+), 194 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 4b862e2eea336..8c9b0b9af5256 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1442,8 +1442,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_offload { - builder.ensure(llvm::Openmp { target }); - builder.ensure(llvm::Offload { target }); + builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 8673e3d7e017f..37640c3b32206 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -911,23 +911,24 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { } #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Openmp { +pub struct OmpOffload { pub target: TargetSelection, } -impl Step for Openmp { +impl Step for OmpOffload { type Output = PathBuf; const IS_HOST: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/openmp") + run.path("src/llvm-project/offload") } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Openmp { target: run.target }); + run.builder.ensure(OmpOffload { target: run.target }); } /// Compile OpenMP offload runtimes for `target`. + #[allow(unused)] fn run(self, builder: &Builder<'_>) -> PathBuf { if builder.config.dry_run() { return PathBuf::from("/"); @@ -941,7 +942,7 @@ impl Step for Openmp { //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { // generate_smart_stamp_hash( // builder, - // &builder.config.src.join("src/llvm-project/openmp"), + // &builder.config.src.join("src/llvm-project/offload"), // builder.offload_info.sha().unwrap_or_default(), // ) //}); @@ -963,8 +964,8 @@ impl Step for Openmp { // return out_dir; //} - trace!(?target, "(re)building openmp artifacts"); - builder.info(&format!("Building OpenMP for {target}")); + trace!(?target, "(re)building offload artifacts"); + builder.info(&format!("Building Omp/Offload for {target}")); //t!(stamp.remove()); let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); @@ -987,10 +988,7 @@ impl Step for Openmp { .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) .define("LLVM_ENABLE_ASSERTIONS", "ON") - .define("TARGET_TRIPLE", &target.triple) - .define("LLVM_ENABLE_RUNTIMES", "offload") - //.define("CMAKE_C_COMPILER", builder.cc(target)) - //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") //.define( // "CMAKE_C_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", @@ -999,197 +997,20 @@ impl Step for Openmp { // "CMAKE_CXX_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", //) + .define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) - .define("OFFLOAD_STANDALONE_BUILD", "ON") - .define("LLVM_ROOT", builder.llvm_out(target)) + .define("OPENMP_STANDALONE_BUILD", "ON") + .define("LLVM_ROOT", builder.llvm_out(target).join("build")) //.define( // "LLVM_ROOT", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", //) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - - if builder.config.llvm_clang { - // This is likely the case locally. If we just build clang in the previous step, we - // should use it. We wouldn't even need a standalone build here, but it's easier to - // unify this with the case below, which needs standalone builds. - // We likely wouldn't need to specify these here, they should be found by default. - cfg.define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); - } else { - // This is the (more complicated) case which we have e.g. in CI. We first build a - // standalone llvm/llvm-project including clang. Then we use this clang to build the - // llvm submodule of rustc in src/llvm-project. In this case we do *NOT* build clang - // again in the llvm submodule to save compile times, this is a hard requirement. - // If we build the offload and openmp runtimes in-tree, they assume that clang was - // already build in-tree, which is not the case here. We therefore must use standalone - // builds of these two runtimes. We now have to specify clang(++) manually to match the - // one we build in llvm/llvm-project - cfg.define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); - } - // /tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ - // -DCMAKE_BUILD_TYPE=Release cfg.build(); //t!(stamp.write()); out_dir - - //$> cd llvm-project # The llvm-project checkout - //$> mkdir build # A different build directory for the build tools - //$> cd build - //$> TARGET_TRIPLE= - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> ninja install - // - //You can do the same thing for the offload project. - // - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DCMAKE_BUILD_TYPE=Release - //$> ninja install - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Offload { - pub target: TargetSelection, -} - -impl Step for Offload { - type Output = PathBuf; - const IS_HOST: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/offload") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Offload { target: run.target }); - } - - /// Compile OpenMP offload runtimes for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::from("/"); - } - let target = self.target; - - let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); - - let out_dir = builder.llvm_out(target); - static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - // generate_smart_stamp_hash( - // builder, - // &builder.config.src.join("src/llvm-project/offload"), - // builder.offload_info.sha().unwrap_or_default(), - // ) - //}); - //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); - - trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); - //if stamp.is_up_to_date() { - // trace!(?out_dir, "offload/openmp build artifacts are up to date"); - // if stamp.stamp().is_empty() { - // builder.info( - // "Could not determine the Offload submodule commit hash. \ - // Assuming that an Offload rebuild is not necessary.", - // ); - // builder.info(&format!( - // "To force Offload to rebuild, remove the file `{}`", - // stamp.path().display() - // )); - // } - // return out_dir; - //} - - trace!(?target, "(re)building offload artifacts"); - builder.info(&format!("Building Offload for {target}")); - //t!(stamp.remove()); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - - builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); - let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); - configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); - - // Re-use the same flags as llvm to control the level of debug information - // generated by Enzyme. - // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - trace!(?profile); - - cfg.out_dir(&out_dir) - .profile(profile) - .env("LLVM_CONFIG_REAL", &host_llvm_config) - .define("LLVM_ENABLE_ASSERTIONS", "ON") - .define("TARGET_TRIPLE", &target.triple) - .define("LLVM_ENABLE_RUNTIMES", "openmp") - .define( - "CMAKE_C_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", - ) - .define( - "CMAKE_CXX_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", - ) - //.define("CMAKE_C_COMPILER", builder.cc(target)) - //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) - .define("OPENMP_STANDALONE_BUILD", "ON") - .define( - "LLVM_ROOT", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", - ) - .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ - // -DCMAKE_BUILD_TYPE=Release - cfg.build(); - - //t!(stamp.write()); - out_dir - - //$> cd llvm-project # The llvm-project checkout - //$> mkdir build # A different build directory for the build tools - //$> cd build - //$> TARGET_TRIPLE= - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> ninja install - // - //You can do the same thing for the offload project. - // - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DCMAKE_BUILD_TYPE=Release - //$> ninja install } } From 7451185ad5000d1413ec5516ac3349ebebe20e25 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 1 Dec 2025 20:14:51 -0800 Subject: [PATCH 22/48] disable default triple, should likely bei either amdgcn or nvptx and we don't know which --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 37640c3b32206..98e071be593b7 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -999,7 +999,7 @@ impl Step for OmpOffload { //) .define("CMAKE_C_COMPILER", builder.cc(target)) .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + //.define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", builder.llvm_out(target).join("build")) //.define( From 0b73ee55511429101020ebd7a771336bc65f3cfa Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 16:55:19 -0800 Subject: [PATCH 23/48] allow specifying cc/cxx for llvm_offload subproject build --- src/bootstrap/src/core/build_steps/llvm.rs | 68 +++++++++++++--------- src/bootstrap/src/core/config/config.rs | 10 ++++ src/bootstrap/src/core/config/toml/llvm.rs | 4 ++ src/bootstrap/src/lib.rs | 5 ++ 4 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 98e071be593b7..5865e94d888f2 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -939,34 +939,34 @@ impl Step for OmpOffload { let out_dir = builder.llvm_out(target); static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - // generate_smart_stamp_hash( - // builder, - // &builder.config.src.join("src/llvm-project/offload"), - // builder.offload_info.sha().unwrap_or_default(), - // ) - //}); - //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); + let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + generate_smart_stamp_hash( + builder, + &builder.config.src.join("src/llvm-project/offload"), + builder.offload_info.sha().unwrap_or_default(), + ) + }); + let stamp = BuildStamp::new(&out_dir).with_prefix("offload").add_stamp(smart_stamp_hash); trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); - //if stamp.is_up_to_date() { - // trace!(?out_dir, "offload/openmp build artifacts are up to date"); - // if stamp.stamp().is_empty() { - // builder.info( - // "Could not determine the Offload submodule commit hash. \ - // Assuming that an Offload rebuild is not necessary.", - // ); - // builder.info(&format!( - // "To force Offload to rebuild, remove the file `{}`", - // stamp.path().display() - // )); - // } - // return out_dir; - //} - - trace!(?target, "(re)building offload artifacts"); - builder.info(&format!("Building Omp/Offload for {target}")); - //t!(stamp.remove()); + if stamp.is_up_to_date() { + trace!(?out_dir, "offload/openmp build artifacts are up to date"); + if stamp.stamp().is_empty() { + builder.info( + "Could not determine the Offload submodule commit hash. \ + Assuming that an Offload rebuild is not necessary.", + ); + builder.info(&format!( + "To force Offload/OpenMP to rebuild, remove the file `{}`", + stamp.path().display() + )); + } + return out_dir; + } + + trace!(?target, "(re)building offload/openmp artifacts"); + builder.info(&format!("Building OpenMP/Offload for {target}")); + t!(stamp.remove()); let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); @@ -984,6 +984,17 @@ impl Step for OmpOffload { }; trace!(?profile); + let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { + p.clone() + } else { + builder.cc(target) + }; + let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { + p.clone() + } else { + builder.cxx(target).unwrap() + }; + cfg.out_dir(&out_dir) .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) @@ -997,9 +1008,8 @@ impl Step for OmpOffload { // "CMAKE_CXX_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", //) - .define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - //.define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + .define("CMAKE_C_COMPILER", cc) + .define("CMAKE_CXX_COMPILER", cxx) .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", builder.llvm_out(target).join("build")) //.define( diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 2f493658ec0ec..e3eb6fb6a2f91 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -169,6 +169,8 @@ pub struct Config { pub llvm_link_jobs: Option, pub llvm_version_suffix: Option, pub llvm_use_linker: Option, + pub llvm_offload_cc: Option, + pub llvm_offload_cxx: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, @@ -290,6 +292,7 @@ pub struct Config { pub miri_info: channel::GitInfo, pub rustfmt_info: channel::GitInfo, pub enzyme_info: channel::GitInfo, + pub offload_info: channel::GitInfo, pub in_tree_llvm_info: channel::GitInfo, pub in_tree_gcc_info: channel::GitInfo, @@ -601,6 +604,8 @@ impl Config { ldflags: llvm_ldflags, use_libcxx: llvm_use_libcxx, use_linker: llvm_use_linker, + offload_cc: llvm_offload_cc, + offload_cxx: llvm_offload_cxx, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, polly: llvm_polly, @@ -1257,6 +1262,8 @@ impl Config { let in_tree_gcc_info = git_info(&exec_ctx, false, &src.join("src/gcc")); let in_tree_llvm_info = git_info(&exec_ctx, false, &src.join("src/llvm-project")); let enzyme_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/enzyme")); + let offload_info = + git_info(&exec_ctx, omit_git_hash, &src.join("src/llvm-project/offload")); let miri_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/miri")); let rust_analyzer_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/rust-analyzer")); @@ -1320,6 +1327,7 @@ impl Config { ehcont_guard: rust_ehcont_guard.unwrap_or(false), enable_bolt_settings: flags_enable_bolt_settings, enzyme_info, + offload_info, exec_ctx, explicit_stage_from_cli: flags_stage.is_some(), explicit_stage_from_config, @@ -1360,6 +1368,8 @@ impl Config { llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), llvm_experimental_targets, + llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), + llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 9523f80214849..30eaa67fc4725 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -32,6 +32,8 @@ define_config! { ldflags: Option = "ldflags", use_libcxx: Option = "use-libcxx", use_linker: Option = "use-linker", + offload_cc: Option = "offload-cc", + offload_cxx: Option = "offload-cxx", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", polly: Option = "polly", @@ -110,6 +112,8 @@ pub fn check_incompatible_options_for_ci_llvm( ldflags, use_libcxx, use_linker, + offload_cc, + offload_cxx, allow_old_toolchain, offload, polly, diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index a31eb0c1c8012..d882c7619ab93 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -212,12 +212,15 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, + //offload_cc: PathBuf, + //offload_cxx: PathBuf, cargo_info: GitInfo, rust_analyzer_info: GitInfo, clippy_info: GitInfo, miri_info: GitInfo, rustfmt_info: GitInfo, enzyme_info: GitInfo, + offload_info: GitInfo, in_tree_llvm_info: GitInfo, in_tree_gcc_info: GitInfo, local_rebuild: bool, @@ -476,6 +479,7 @@ impl Build { let miri_info = config.miri_info.clone(); let rustfmt_info = config.rustfmt_info.clone(); let enzyme_info = config.enzyme_info.clone(); + let offload_info = config.offload_info.clone(); let in_tree_llvm_info = config.in_tree_llvm_info.clone(); let in_tree_gcc_info = config.in_tree_gcc_info.clone(); @@ -568,6 +572,7 @@ impl Build { miri_info, rustfmt_info, enzyme_info, + offload_info, in_tree_llvm_info, in_tree_gcc_info, cc: HashMap::new(), From ed519d16b369bd8136b17a7b441923563f2cb8a9 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 16:59:19 -0800 Subject: [PATCH 24/48] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 23 ---------------------- 1 file changed, 23 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 5865e94d888f2..761c8c8ada630 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -455,29 +455,6 @@ impl Step for Llvm { enabled_llvm_runtimes.push("compiler-rt"); } - // This is an experimental flag, which likely builds more than necessary. - // We will optimize it when we get closer to releasing it on nightly. - if builder.config.llvm_offload { - //enabled_llvm_projects.push("lld"); - //enabled_llvm_runtimes.push("offload"); - //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. - //Remove this line once they achieved it. - //enabled_llvm_runtimes.push("openmp"); - //enabled_llvm_runtimes.push("compiler-rt"); - - // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; - //cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); - //cfg.define("LLVM_TARGETS_TO_BUILD", "host;AMDGPU;NVPTX"); - - //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; - //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); - - //cfg.define("LLVM_INCLUDE_TESTS", "OFF"); - //cfg.define("LLVM_BUILD_TESTS", "OFF"); - //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); - //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); - } - if !enabled_llvm_projects.is_empty() { enabled_llvm_projects.sort(); enabled_llvm_projects.dedup(); From 56bf506c5f3cf2966e8d6630b1d9cabc978c2fe9 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 19:50:09 -0800 Subject: [PATCH 25/48] worked locally with setting LLVM_ROOT and cc/cxx --- src/bootstrap/src/core/build_steps/llvm.rs | 13 +++++++++++-- src/bootstrap/src/core/config/config.rs | 3 +++ src/bootstrap/src/core/config/toml/llvm.rs | 2 ++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 761c8c8ada630..bf08fee4f2704 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -971,12 +971,20 @@ impl Step for OmpOffload { } else { builder.cxx(target).unwrap() }; + let root = if let Some(p) = &builder.build.config.llvm_root_offload { + p.clone() + } else { + builder.llvm_out(target).join("build") + }; cfg.out_dir(&out_dir) .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") + .define("LLVM_INCLUDE_TESTS", "OFF") + .define("LLVM_BUILD_TESTS", "OFF") + .define("OFFLOAD_INCLUDE_TESTS", "OFF") //.define( // "CMAKE_C_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", @@ -987,8 +995,9 @@ impl Step for OmpOffload { //) .define("CMAKE_C_COMPILER", cc) .define("CMAKE_CXX_COMPILER", cxx) + .define("Clang_DIR", "/tmp/drehwald1/prog/llvm/lib/cmake/clang") .define("OPENMP_STANDALONE_BUILD", "ON") - .define("LLVM_ROOT", builder.llvm_out(target).join("build")) + .define("LLVM_ROOT", root) //.define( // "LLVM_ROOT", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", @@ -996,7 +1005,7 @@ impl Step for OmpOffload { .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); cfg.build(); - //t!(stamp.write()); + t!(stamp.write()); out_dir } } diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index e3eb6fb6a2f91..90013a3b7e0a9 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -171,6 +171,7 @@ pub struct Config { pub llvm_use_linker: Option, pub llvm_offload_cc: Option, pub llvm_offload_cxx: Option, + pub llvm_root_offload: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, @@ -606,6 +607,7 @@ impl Config { use_linker: llvm_use_linker, offload_cc: llvm_offload_cc, offload_cxx: llvm_offload_cxx, + root_offload: llvm_root_offload, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, polly: llvm_polly, @@ -1370,6 +1372,7 @@ impl Config { llvm_experimental_targets, llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), + llvm_root_offload: llvm_root_offload.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 30eaa67fc4725..43252ceea08a4 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -34,6 +34,7 @@ define_config! { use_linker: Option = "use-linker", offload_cc: Option = "offload-cc", offload_cxx: Option = "offload-cxx", + root_offload: Option = "root-offload", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", polly: Option = "polly", @@ -114,6 +115,7 @@ pub fn check_incompatible_options_for_ci_llvm( use_linker, offload_cc, offload_cxx, + root_offload, allow_old_toolchain, offload, polly, From d384a01cd5d0ad87d372c58c506332378d0c915e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:21:01 -0800 Subject: [PATCH 26/48] new try --- src/bootstrap/src/core/build_steps/llvm.rs | 46 ++++++++++------------ src/bootstrap/src/core/config/config.rs | 3 ++ src/bootstrap/src/core/config/toml/llvm.rs | 1 + 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index bf08fee4f2704..d7e7973ee77ff 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -961,21 +961,28 @@ impl Step for OmpOffload { }; trace!(?profile); - let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { - p.clone() - } else { - builder.cc(target) - }; - let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { - p.clone() - } else { - builder.cxx(target).unwrap() - }; + //let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { + // //p.clone() + // builder.cc(target) + //} else { + // builder.cc(target) + //}; + //let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { + // //p.clone() + // builder.cxx(target).unwrap() + //} else { + // builder.cxx(target).unwrap() + //}; let root = if let Some(p) = &builder.build.config.llvm_root_offload { p.clone() } else { builder.llvm_out(target).join("build") }; + let clang_dir = if !builder.config.llvm_clang { + &builder.build.config.llvm_clang_dir.unwrap() + } else { + None + }; cfg.out_dir(&out_dir) .profile(profile) @@ -983,26 +990,15 @@ impl Step for OmpOffload { .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") .define("LLVM_INCLUDE_TESTS", "OFF") - .define("LLVM_BUILD_TESTS", "OFF") .define("OFFLOAD_INCLUDE_TESTS", "OFF") - //.define( - // "CMAKE_C_COMPILER", - // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", - //) - //.define( - // "CMAKE_CXX_COMPILER", - // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", - //) - .define("CMAKE_C_COMPILER", cc) - .define("CMAKE_CXX_COMPILER", cxx) + //.define("CMAKE_C_COMPILER", cc) + //.define("CMAKE_CXX_COMPILER", cxx) .define("Clang_DIR", "/tmp/drehwald1/prog/llvm/lib/cmake/clang") .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", root) - //.define( - // "LLVM_ROOT", - // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", - //) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + if let Some(p) = Clang_DIR { + cfg.define("CLANG_DIR", p); cfg.build(); t!(stamp.write()); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 90013a3b7e0a9..80b72a0afeef1 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -172,6 +172,7 @@ pub struct Config { pub llvm_offload_cc: Option, pub llvm_offload_cxx: Option, pub llvm_root_offload: Option, + pub llvm_clang_dir: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, @@ -608,6 +609,7 @@ impl Config { offload_cc: llvm_offload_cc, offload_cxx: llvm_offload_cxx, root_offload: llvm_root_offload, + clang_dir: llvm_clang_dir, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, polly: llvm_polly, @@ -1373,6 +1375,7 @@ impl Config { llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_root_offload: llvm_root_offload.map(PathBuf::from), + llvm_clang_dir: llvm_clang_dir.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 43252ceea08a4..17cd9c7b33bdc 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -35,6 +35,7 @@ define_config! { offload_cc: Option = "offload-cc", offload_cxx: Option = "offload-cxx", root_offload: Option = "root-offload", + clang_dir: Option = "offload-clang-dir", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", polly: Option = "polly", From 183418ef8820f5c3977ad04f32536ca4316a2c04 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:22:10 -0800 Subject: [PATCH 27/48] new try --- src/bootstrap/src/core/build_steps/llvm.rs | 8 ++++++-- src/bootstrap/src/core/config/config.rs | 4 ++-- src/bootstrap/src/core/config/toml/llvm.rs | 7 ++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d7e7973ee77ff..efe2912d37ee3 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -979,8 +979,11 @@ impl Step for OmpOffload { builder.llvm_out(target).join("build") }; let clang_dir = if !builder.config.llvm_clang { - &builder.build.config.llvm_clang_dir.unwrap() + // We must have an external clang to use. + assert!(&builder.build.config.llvm_clang_dir.is_some()); + builder.build.config.llvm_clang_dir.clone() } else { + // No need to specify it, since we use the in-tree clang None }; @@ -997,8 +1000,9 @@ impl Step for OmpOffload { .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", root) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - if let Some(p) = Clang_DIR { + if let Some(p) = clang_dir { cfg.define("CLANG_DIR", p); + } cfg.build(); t!(stamp.write()); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 80b72a0afeef1..64233ac9e999e 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -1331,7 +1331,6 @@ impl Config { ehcont_guard: rust_ehcont_guard.unwrap_or(false), enable_bolt_settings: flags_enable_bolt_settings, enzyme_info, - offload_info, exec_ctx, explicit_stage_from_cli: flags_stage.is_some(), explicit_stage_from_config, @@ -1368,6 +1367,7 @@ impl Config { llvm_cflags, llvm_clang: llvm_clang.unwrap_or(false), llvm_clang_cl, + llvm_clang_dir: llvm_clang_dir.map(PathBuf::from), llvm_cxxflags, llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), @@ -1375,7 +1375,6 @@ impl Config { llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_root_offload: llvm_root_offload.map(PathBuf::from), - llvm_clang_dir: llvm_clang_dir.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind @@ -1412,6 +1411,7 @@ impl Config { musl_root: rust_musl_root.map(PathBuf::from), ninja_in_file: llvm_ninja.unwrap_or(true), nodejs: build_nodejs.map(PathBuf::from), + offload_info, omit_git_hash, on_fail: flags_on_fail, optimized_compiler_builtins, diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 17cd9c7b33bdc..a83df8d865c71 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -114,9 +114,10 @@ pub fn check_incompatible_options_for_ci_llvm( ldflags, use_libcxx, use_linker, - offload_cc, - offload_cxx, - root_offload, + offload_cc: _, + offload_cxx: _, + root_offload: _, + clang_dir: _, allow_old_toolchain, offload, polly, From 67341b6e5e269134d51186fc1c6f95d499be9963 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:42:19 -0800 Subject: [PATCH 28/48] new try --- src/bootstrap/src/core/build_steps/llvm.rs | 3 +-- src/bootstrap/src/core/config/config.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index efe2912d37ee3..a04138d31e0ae 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -996,12 +996,11 @@ impl Step for OmpOffload { .define("OFFLOAD_INCLUDE_TESTS", "OFF") //.define("CMAKE_C_COMPILER", cc) //.define("CMAKE_CXX_COMPILER", cxx) - .define("Clang_DIR", "/tmp/drehwald1/prog/llvm/lib/cmake/clang") .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", root) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); if let Some(p) = clang_dir { - cfg.define("CLANG_DIR", p); + cfg.define("Clang_DIR", p); } cfg.build(); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 64233ac9e999e..70774eb674535 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -1372,8 +1372,6 @@ impl Config { llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), llvm_experimental_targets, - llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), - llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_root_offload: llvm_root_offload.map(PathBuf::from), llvm_from_ci, llvm_ldflags, @@ -1389,6 +1387,8 @@ impl Config { .or((!llvm_from_ci && llvm_thin_lto.unwrap_or(false)).then_some(true)), ), llvm_offload: llvm_offload.unwrap_or(false), + llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), + llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_optimize: llvm_optimize.unwrap_or(true), llvm_plugins: llvm_plugin.unwrap_or(false), llvm_polly: llvm_polly.unwrap_or(false), From 139b1074600d71704d452b3c176866c98fa1e2d4 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:47:50 -0800 Subject: [PATCH 29/48] try to guess the Clang_DIR to ClangConfig.cmake --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index ba62750643d4c..0a8ecb4d75941 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -91,6 +91,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set llvm.offload=true \ + --set llvm.offload-clang-dir="/rustroot/lib/cmake/clang" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 2271b97643e68ee8cc4de16cc4019c98413c3bb0 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 11 Dec 2025 02:09:19 -0800 Subject: [PATCH 30/48] fix new arg parsing --- bootstrap.example.toml | 3 +++ src/bootstrap/configure.py | 1 + 2 files changed, 4 insertions(+) diff --git a/bootstrap.example.toml b/bootstrap.example.toml index 4e850810a30a9..da0ff3b868480 100644 --- a/bootstrap.example.toml +++ b/bootstrap.example.toml @@ -106,6 +106,9 @@ # Whether to build LLVM with support for it's gpu offload runtime. #llvm.offload = false +# absolute path to ClangConfig.cmake +#llvm.offload-clang-dir = "" + # When true, link libstdc++ statically into the rustc_llvm. # This is useful if you don't want to use the dynamic version of that # library provided by LLVM. diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 1915986be28c5..b6df885056087 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -120,6 +120,7 @@ def v(*args): o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") +o("llvm-offload-clang-dir", "llvm.offload-clang-dir", "pass the absolute directory of ClangConfig.cmake") o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") o( From b1217d1de091e90bb0c187c1f3b196983bac47e6 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 11 Dec 2025 08:16:27 -0800 Subject: [PATCH 31/48] explicitely add llvm Include dir to overwrite out-of-tree clang's default --- src/bootstrap/src/core/build_steps/llvm.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index a04138d31e0ae..1884330b40446 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -779,6 +779,10 @@ fn configure_cmake( if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } + // Manuel + let base = builder.llvm_out(target).join("include"); + let inc_dir = base.display(); + cflags.push(format!(" -I {inc_dir}")); cfg.define("CMAKE_C_FLAGS", cflags); let mut cxxflags: OsString = builder .cc_handled_clags(target, CLang::Cxx) @@ -792,6 +796,7 @@ fn configure_cmake( .collect::>() .join(" ") .into(); + cxxflags.push(format!(" -I {inc_dir}")); if let Some(ref s) = builder.config.llvm_cxxflags { cxxflags.push(" "); cxxflags.push(s); From 95a369bc0d3e84d7c8a5fd4959f2ad80f4e80140 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 11 Dec 2025 10:07:48 -0800 Subject: [PATCH 32/48] try not building omp/offload if pgo is enabled --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 8c9b0b9af5256..5662003890da3 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1441,7 +1441,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload { + if builder.config.llvm_offload && !builder.config.llvm_profile_generate { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } From ae68c356ddb9b9f274bb28e1481fe60a6a38e591 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 08:14:51 -0800 Subject: [PATCH 33/48] use out subdir to prevent artifact deletions --- src/bootstrap/src/core/build_steps/llvm.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 1884330b40446..5d10052b8432a 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -913,19 +913,23 @@ impl Step for OmpOffload { #[allow(unused)] fn run(self, builder: &Builder<'_>) -> PathBuf { if builder.config.dry_run() { - return PathBuf::from("/"); + return builder.config.tempdir().join("llvm-offload-dry-run"); } let target = self.target; let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); - let out_dir = builder.llvm_out(target); + let out_dir = builder.llvm_out(target).join("offload-outdir"); + if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { + std::fs::DirBuilder::new().create(&out_dir).unwrap(); + dbg!("Created out subdir!"); + } static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { generate_smart_stamp_hash( builder, &builder.config.src.join("src/llvm-project/offload"), - builder.offload_info.sha().unwrap_or_default(), + builder.in_tree_llvm_info.sha().unwrap_or_default(), ) }); let stamp = BuildStamp::new(&out_dir).with_prefix("offload").add_stamp(smart_stamp_hash); @@ -952,7 +956,7 @@ impl Step for OmpOffload { let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); - builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); + builder.config.update_submodule("src/llvm-project"); let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); From e82a40844a05ce879aa03a5095794bdb5f7d7191 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 08:46:28 -0800 Subject: [PATCH 34/48] remove outdated offload_info --- src/bootstrap/src/lib.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index d882c7619ab93..a31eb0c1c8012 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -212,15 +212,12 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, - //offload_cc: PathBuf, - //offload_cxx: PathBuf, cargo_info: GitInfo, rust_analyzer_info: GitInfo, clippy_info: GitInfo, miri_info: GitInfo, rustfmt_info: GitInfo, enzyme_info: GitInfo, - offload_info: GitInfo, in_tree_llvm_info: GitInfo, in_tree_gcc_info: GitInfo, local_rebuild: bool, @@ -479,7 +476,6 @@ impl Build { let miri_info = config.miri_info.clone(); let rustfmt_info = config.rustfmt_info.clone(); let enzyme_info = config.enzyme_info.clone(); - let offload_info = config.offload_info.clone(); let in_tree_llvm_info = config.in_tree_llvm_info.clone(); let in_tree_gcc_info = config.in_tree_gcc_info.clone(); @@ -572,7 +568,6 @@ impl Build { miri_info, rustfmt_info, enzyme_info, - offload_info, in_tree_llvm_info, in_tree_gcc_info, cc: HashMap::new(), From 8ee35be325a2f24df6ebf7a00fe9b946fdda8c18 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 10:54:40 -0800 Subject: [PATCH 35/48] cleanup --- src/bootstrap/configure.py | 6 ++- src/bootstrap/src/core/build_steps/llvm.rs | 46 +++++++++++----------- src/bootstrap/src/core/config/config.rs | 9 ----- src/bootstrap/src/core/config/toml/llvm.rs | 6 --- 4 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index b6df885056087..18f6391d2022e 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -120,7 +120,11 @@ def v(*args): o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") -o("llvm-offload-clang-dir", "llvm.offload-clang-dir", "pass the absolute directory of ClangConfig.cmake") +o( + "llvm-offload-clang-dir", + "llvm.offload-clang-dir", + "pass the absolute directory of ClangConfig.cmake", +) o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") o( diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 5d10052b8432a..c56a44c5427a5 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -892,6 +892,17 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { .or_else(|| env::var_os(var_base)) } +// FIXME(offload): In an ideal world, we would just enable the offload runtime in our previous LLVM +// build step. For now, we still depend on the openmp runtime since we use some of it's API, so we +// build both. However, when building those runtimes as part of the LLVM step, then LLVM's cmake +// implicitely assumes that Clang has also been build and will try to use it. In the Rust CI, we +// don't always build clang (due to compile times), but instead use a slightly older external clang. +// LLVM tries to remove this build dependency of offload/openmp on Clang for LLVM-22, so in the +// future we might be able to integrate this step into the LLVM step. For now, we instead introduce +// a Clang_DIR bootstrap option, which allows us tell CMake to use an external clang for these two +// runtimes. This external clang will try to use it's own (older) include dirs when building our +// in-tree LLVM submodule, which will cause build failures. To prevent those, we now also +// explicitely set our include dirs. #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct OmpOffload { pub target: TargetSelection, @@ -919,11 +930,17 @@ impl Step for OmpOffload { let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + // Running cmake twice in the same folder is known to cause issues, like deleting existing + // binaries. We therefore write our offload artifacts into it's own subfolder. We use a + // subfolder, so that all the logic that processes our build artifacts (hopefully) also + // automatically manages our artifacts in the subfolder. let out_dir = builder.llvm_out(target).join("offload-outdir"); if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { std::fs::DirBuilder::new().create(&out_dir).unwrap(); dbg!("Created out subdir!"); } + + // Offload/OpenMP are just subfolders of LLVM, so we can use the LLVM sha. static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { generate_smart_stamp_hash( @@ -961,8 +978,7 @@ impl Step for OmpOffload { configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); // Re-use the same flags as llvm to control the level of debug information - // generated by Enzyme. - // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. + // generated for offload. let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { (false, _) => "Debug", (true, false) => "Release", @@ -970,23 +986,8 @@ impl Step for OmpOffload { }; trace!(?profile); - //let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { - // //p.clone() - // builder.cc(target) - //} else { - // builder.cc(target) - //}; - //let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { - // //p.clone() - // builder.cxx(target).unwrap() - //} else { - // builder.cxx(target).unwrap() - //}; - let root = if let Some(p) = &builder.build.config.llvm_root_offload { - p.clone() - } else { - builder.llvm_out(target).join("build") - }; + // OpenMP/Offload builds currently (LLVM-21) still depend on Clang, although there are + // intentions to loosen this requirement for LLVM-22. If we were to let clang_dir = if !builder.config.llvm_clang { // We must have an external clang to use. assert!(&builder.build.config.llvm_clang_dir.is_some()); @@ -996,6 +997,9 @@ impl Step for OmpOffload { None }; + // FIXME(offload): Once we move from OMP to Offload (Ol) APIs, we should drop the openmp + // runtime to simplify our build. We should also re-evaluate the LLVM_Root and try to get + // rid of the Clang_DIR, once we upgrade to LLVM-22. cfg.out_dir(&out_dir) .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) @@ -1003,10 +1007,8 @@ impl Step for OmpOffload { .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") .define("LLVM_INCLUDE_TESTS", "OFF") .define("OFFLOAD_INCLUDE_TESTS", "OFF") - //.define("CMAKE_C_COMPILER", cc) - //.define("CMAKE_CXX_COMPILER", cxx) .define("OPENMP_STANDALONE_BUILD", "ON") - .define("LLVM_ROOT", root) + .define("LLVM_ROOT", builder.llvm_out(target).join("build")) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); if let Some(p) = clang_dir { cfg.define("Clang_DIR", p); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 70774eb674535..a2b2836ac743e 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -169,9 +169,6 @@ pub struct Config { pub llvm_link_jobs: Option, pub llvm_version_suffix: Option, pub llvm_use_linker: Option, - pub llvm_offload_cc: Option, - pub llvm_offload_cxx: Option, - pub llvm_root_offload: Option, pub llvm_clang_dir: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, @@ -606,9 +603,6 @@ impl Config { ldflags: llvm_ldflags, use_libcxx: llvm_use_libcxx, use_linker: llvm_use_linker, - offload_cc: llvm_offload_cc, - offload_cxx: llvm_offload_cxx, - root_offload: llvm_root_offload, clang_dir: llvm_clang_dir, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, @@ -1372,7 +1366,6 @@ impl Config { llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), llvm_experimental_targets, - llvm_root_offload: llvm_root_offload.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind @@ -1387,8 +1380,6 @@ impl Config { .or((!llvm_from_ci && llvm_thin_lto.unwrap_or(false)).then_some(true)), ), llvm_offload: llvm_offload.unwrap_or(false), - llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), - llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_optimize: llvm_optimize.unwrap_or(true), llvm_plugins: llvm_plugin.unwrap_or(false), llvm_polly: llvm_polly.unwrap_or(false), diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index a83df8d865c71..8f42c0027aa5f 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -32,9 +32,6 @@ define_config! { ldflags: Option = "ldflags", use_libcxx: Option = "use-libcxx", use_linker: Option = "use-linker", - offload_cc: Option = "offload-cc", - offload_cxx: Option = "offload-cxx", - root_offload: Option = "root-offload", clang_dir: Option = "offload-clang-dir", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", @@ -114,9 +111,6 @@ pub fn check_incompatible_options_for_ci_llvm( ldflags, use_libcxx, use_linker, - offload_cc: _, - offload_cxx: _, - root_offload: _, clang_dir: _, allow_old_toolchain, offload, From a977d3bb571b2610363261f7b2b20ab1256aea1e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 11:06:07 -0800 Subject: [PATCH 36/48] cleanup --- src/bootstrap/configure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 18f6391d2022e..28fd9cf572d68 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -121,9 +121,9 @@ def v(*args): o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") o( - "llvm-offload-clang-dir", - "llvm.offload-clang-dir", - "pass the absolute directory of ClangConfig.cmake", + "llvm-offload-clang-dir", + "llvm.offload-clang-dir", + "pass the absolute directory of ClangConfig.cmake", ) o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") From 119fc174f6a84c0c93cb4334cee17755834433b2 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 11:16:04 -0800 Subject: [PATCH 37/48] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index c56a44c5427a5..37d92514bd02c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -895,7 +895,7 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { // FIXME(offload): In an ideal world, we would just enable the offload runtime in our previous LLVM // build step. For now, we still depend on the openmp runtime since we use some of it's API, so we // build both. However, when building those runtimes as part of the LLVM step, then LLVM's cmake -// implicitely assumes that Clang has also been build and will try to use it. In the Rust CI, we +// implicitly assumes that Clang has also been build and will try to use it. In the Rust CI, we // don't always build clang (due to compile times), but instead use a slightly older external clang. // LLVM tries to remove this build dependency of offload/openmp on Clang for LLVM-22, so in the // future we might be able to integrate this step into the LLVM step. For now, we instead introduce From 7895f56f0c1ac2ccfef8fd9e3442f37a320833d0 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 12:49:35 -0800 Subject: [PATCH 38/48] typo --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 37d92514bd02c..d4aa08bcd0fbb 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -902,7 +902,7 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { // a Clang_DIR bootstrap option, which allows us tell CMake to use an external clang for these two // runtimes. This external clang will try to use it's own (older) include dirs when building our // in-tree LLVM submodule, which will cause build failures. To prevent those, we now also -// explicitely set our include dirs. +// explicitly set our include dirs. #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct OmpOffload { pub target: TargetSelection, From e460234176a25d88475afbd4a7daba2f8fc74bfe Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 12:59:02 -0800 Subject: [PATCH 39/48] clean up Include dir handling --- src/bootstrap/src/core/build_steps/llvm.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d4aa08bcd0fbb..a6861b45f9531 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -769,6 +769,17 @@ fn configure_cmake( .collect::>() .join(" ") .into(); + + // If we use an external clang as opposed to building our own llvm_clang, than that clang will + // come with it's own set of default include directories, which are based on a potentially older + // LLVM. This can cause issues, so we overwrite it to include headers based on our + // `src/llvm-project` submodule instead. + // FIXME(offload): With LLVM-22 we hopefully won't need an external clang anymore. + let base = builder.llvm_out(target).join("include"); + let inc_dir = base.display(); + if builder.config.llvm_offload && !builder.config.llvm_clang { + cflags.push(format!(" -I {inc_dir}")); + } if let Some(ref s) = builder.config.llvm_cflags { cflags.push(" "); cflags.push(s); @@ -779,11 +790,8 @@ fn configure_cmake( if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } - // Manuel - let base = builder.llvm_out(target).join("include"); - let inc_dir = base.display(); - cflags.push(format!(" -I {inc_dir}")); cfg.define("CMAKE_C_FLAGS", cflags); + let mut cxxflags: OsString = builder .cc_handled_clags(target, CLang::Cxx) .into_iter() @@ -796,7 +804,9 @@ fn configure_cmake( .collect::>() .join(" ") .into(); - cxxflags.push(format!(" -I {inc_dir}")); + if builder.config.llvm_offload && !builder.config.llvm_clang { + cxxflags.push(format!(" -I {inc_dir}")); + } if let Some(ref s) = builder.config.llvm_cxxflags { cxxflags.push(" "); cxxflags.push(s); From 260d24b924e230c6c528756c16c214e0ac6af684 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 13:03:58 -0800 Subject: [PATCH 40/48] Update ChangeInfo to account for new offload-clang-dir flag --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 37ad5f09897fa..b9713b93df769 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -601,4 +601,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "New options `rust.rustflags` for all targets and per-target `rustflags` that will pass specified flags to rustc for all stages. Target-specific flags override global `rust.rustflags` ones.", }, + ChangeInfo { + change_id: 148671, + severity: ChangeSeverity::Info, + summary: "New option `llvm.offload-clang-dir` to allow building an in-tree llvm offload and openmp runtime with an external clang.", + }, ]; From 6e83fa1d0c2de8d49ad91dfa1123ff2f3373e2a9 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 13:05:56 -0800 Subject: [PATCH 41/48] remove checking for llvm_profile_generate, since it likely wasn't the culprit --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 5662003890da3..8c9b0b9af5256 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1441,7 +1441,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload && !builder.config.llvm_profile_generate { + if builder.config.llvm_offload { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } From 0d26ab38c67681609d62ea0d528e8b023713fc0c Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 13:15:42 -0800 Subject: [PATCH 42/48] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index a6861b45f9531..83982ff298cf3 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -947,7 +947,6 @@ impl Step for OmpOffload { let out_dir = builder.llvm_out(target).join("offload-outdir"); if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { std::fs::DirBuilder::new().create(&out_dir).unwrap(); - dbg!("Created out subdir!"); } // Offload/OpenMP are just subfolders of LLVM, so we can use the LLVM sha. From 98a2e0fd861dca782b6c00311de7219ef8f9d9f6 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 13 Dec 2025 05:46:29 +0100 Subject: [PATCH 43/48] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 83982ff298cf3..3a8c88325d98c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -945,7 +945,7 @@ impl Step for OmpOffload { // subfolder, so that all the logic that processes our build artifacts (hopefully) also // automatically manages our artifacts in the subfolder. let out_dir = builder.llvm_out(target).join("offload-outdir"); - if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { + if std::fs::exists(&out_dir).is_ok_and(|x| !x) { std::fs::DirBuilder::new().create(&out_dir).unwrap(); } From 4b1cb50e6ad1fff83d36610fbc5fa964048bc4d9 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 13 Dec 2025 05:22:35 -0800 Subject: [PATCH 44/48] wip --- src/bootstrap/src/core/build_steps/compile.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 8c9b0b9af5256..e1fff62aa0df6 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2216,6 +2216,42 @@ impl Step for Assemble { } } + if builder.config.llvm_offload && !builder.config.dry_run() { + let offload_install = builder.ensure(llvm::OmpOffload { target: build_compiler.host }); + if let Some(llvm_config) = builder.llvm_config(builder.config.host_target) { + let src_dir = offload_install.join("lib"); + let target_libdir = + builder.sysroot_target_libdir(target_compiler, target_compiler.host); + let llvm = llvm::get_llvm_version(builder, &llvm_config); + let llvm_version = llvm.split('.').take(2).collect::>().join("."); + let rust_version = builder.rust_version(); + // e.g. libLLVMOffload.so.21.1 + let lib_ext = std::env::consts::DLL_EXTENSION; + let libenzyme = format!("libLLVMOffload.{lib_ext}.{llvm_version}"); + dbg!(&libenzyme); + let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext); + let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext); + let src_lib = src_dir.join(&libenzyme); + builder.copy_link(&src_lib, &dst_lib, FileType::NativeLibrary); + builder.copy_link(&src_lib, &target_dst_lib, FileType::NativeLibrary); + + // libomp.so, libomptarget.so + let omp = src_dir.join("libomp").with_extension(lib_ext); + let tgt = src_dir.join("libomptarget").with_extension(lib_ext); + builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); + builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); + builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); + builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); + //[\u@\h:\W]$ ls ../offload-outdir/lib + // amdgcn-amd-amdhsa libarcher.so libgomp.so libiomp5.so libLLVMOffload.so.21.1 libomp.so libomptarget.so.21.1 + // cmake libarcher_static.a libgomp.so.1 libLLVMOffload.so libompd.so libomptarget.so nvptx64-nvidia-cuda + // [\u@\h:\W]$ ls ../offload-outdir/lib/amdgcn-amd-amdhsa + // libompdevice.a libomptarget-amdgpu.bc + // [\u@\h:\W]$ ls nv + // [\u@\h:\W]$ ls ../offload-outdir/lib/nvptx64-nvidia-cuda + } + } + // Build the libraries for this compiler to link to (i.e., the libraries // it uses at runtime). debug!( From d33c8c7ae9c933d329cc0dd0ac89cfd518120d8a Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 13 Dec 2025 07:18:17 -0800 Subject: [PATCH 45/48] imprv --- src/bootstrap/src/core/build_steps/compile.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index e1fff62aa0df6..2813f2ca0f2a3 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2217,9 +2217,11 @@ impl Step for Assemble { } if builder.config.llvm_offload && !builder.config.dry_run() { + debug!("`llvm_offload` requested"); let offload_install = builder.ensure(llvm::OmpOffload { target: build_compiler.host }); if let Some(llvm_config) = builder.llvm_config(builder.config.host_target) { let src_dir = offload_install.join("lib"); + let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); let target_libdir = builder.sysroot_target_libdir(target_compiler, target_compiler.host); let llvm = llvm::get_llvm_version(builder, &llvm_config); @@ -2228,20 +2230,19 @@ impl Step for Assemble { // e.g. libLLVMOffload.so.21.1 let lib_ext = std::env::consts::DLL_EXTENSION; let libenzyme = format!("libLLVMOffload.{lib_ext}.{llvm_version}"); - dbg!(&libenzyme); - let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext); - let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext); + let dst_lib = libdir.join(&libenzyme); + let target_dst_lib = target_libdir.join(&libenzyme); let src_lib = src_dir.join(&libenzyme); builder.copy_link(&src_lib, &dst_lib, FileType::NativeLibrary); builder.copy_link(&src_lib, &target_dst_lib, FileType::NativeLibrary); // libomp.so, libomptarget.so - let omp = src_dir.join("libomp").with_extension(lib_ext); - let tgt = src_dir.join("libomptarget").with_extension(lib_ext); - builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); - builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); - builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); - builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); + //let omp = src_dir.join("libomp").with_extension(lib_ext); + //let tgt = src_dir.join("libomptarget").with_extension(lib_ext); + //builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); + //builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); + //builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); + //builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); //[\u@\h:\W]$ ls ../offload-outdir/lib // amdgcn-amd-amdhsa libarcher.so libgomp.so libiomp5.so libLLVMOffload.so.21.1 libomp.so libomptarget.so.21.1 // cmake libarcher_static.a libgomp.so.1 libLLVMOffload.so libompd.so libomptarget.so nvptx64-nvidia-cuda From b801e4b0931e9bd4e54653814f8ff241d97f6c96 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 13 Dec 2025 08:09:12 -0800 Subject: [PATCH 46/48] working --- src/bootstrap/src/core/build_steps/compile.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 2813f2ca0f2a3..38d27c4b4aac1 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2229,12 +2229,17 @@ impl Step for Assemble { let rust_version = builder.rust_version(); // e.g. libLLVMOffload.so.21.1 let lib_ext = std::env::consts::DLL_EXTENSION; - let libenzyme = format!("libLLVMOffload.{lib_ext}.{llvm_version}"); - let dst_lib = libdir.join(&libenzyme); - let target_dst_lib = target_libdir.join(&libenzyme); - let src_lib = src_dir.join(&libenzyme); - builder.copy_link(&src_lib, &dst_lib, FileType::NativeLibrary); - builder.copy_link(&src_lib, &target_dst_lib, FileType::NativeLibrary); + let libenzyme = format!("libLLVMOffload"); + //let libenzyme = format!("libLLVMOffload.{lib_ext}.{llvm_version}"); + let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext); + let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext); + let src_lib = src_dir.join(&libenzyme).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_lib, &dst_lib); //, FileType::NativeLibrary + builder.resolve_symlink_and_copy( + &src_lib, + &target_dst_lib, + //FileType::NativeLibrary, + ); // libomp.so, libomptarget.so //let omp = src_dir.join("libomp").with_extension(lib_ext); From 328e0e215183da7989e6b061193096110f4b5b8b Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 13 Dec 2025 09:16:50 -0800 Subject: [PATCH 47/48] imprv --- src/bootstrap/src/core/build_steps/compile.rs | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 38d27c4b4aac1..f4b8f71cd797d 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2224,30 +2224,33 @@ impl Step for Assemble { let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); let target_libdir = builder.sysroot_target_libdir(target_compiler, target_compiler.host); - let llvm = llvm::get_llvm_version(builder, &llvm_config); - let llvm_version = llvm.split('.').take(2).collect::>().join("."); - let rust_version = builder.rust_version(); - // e.g. libLLVMOffload.so.21.1 let lib_ext = std::env::consts::DLL_EXTENSION; + let libenzyme = format!("libLLVMOffload"); - //let libenzyme = format!("libLLVMOffload.{lib_ext}.{llvm_version}"); + let src_lib = src_dir.join(&libenzyme).with_extension(lib_ext); let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext); let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext); - let src_lib = src_dir.join(&libenzyme).with_extension(lib_ext); - builder.resolve_symlink_and_copy(&src_lib, &dst_lib); //, FileType::NativeLibrary - builder.resolve_symlink_and_copy( - &src_lib, - &target_dst_lib, - //FileType::NativeLibrary, - ); - - // libomp.so, libomptarget.so - //let omp = src_dir.join("libomp").with_extension(lib_ext); - //let tgt = src_dir.join("libomptarget").with_extension(lib_ext); - //builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); - //builder.copy_link(&omp, &dst_lib, FileType::NativeLibrary); - //builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); - //builder.copy_link(&tgt, &dst_lib, FileType::NativeLibrary); + builder.resolve_symlink_and_copy(&src_lib, &dst_lib); + builder.resolve_symlink_and_copy(&src_lib, &target_dst_lib); + + // FIXME(offload): With LLVM-22, we should be able to drop everything below here. + let omp = format!("libomp"); + let src_omp = src_dir.join(&omp).with_extension(lib_ext); + let dst_omp_lib = libdir.join(&omp).with_extension(lib_ext); + let target_omp_dst_lib = target_libdir.join(&omp).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_omp, &dst_omp_lib); + builder.resolve_symlink_and_copy(&src_omp, &target_omp_dst_lib); + + let tgt = format!("libomptarget"); + let src_tgt = src_dir.join(&tgt).with_extension(lib_ext); + let dst_tgt_lib = libdir.join(&tgt).with_extension(lib_ext); + let target_tgt_dst_lib = target_libdir.join(&tgt).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); + builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); + + // The last one is slightly more tricky, since we have the same file twice, in two + // subfolders for amdgcn and nvptx64. We'll likely find two more in the future, once + // Intel and Spir-V support lands in offload. //[\u@\h:\W]$ ls ../offload-outdir/lib // amdgcn-amd-amdhsa libarcher.so libgomp.so libiomp5.so libLLVMOffload.so.21.1 libomp.so libomptarget.so.21.1 // cmake libarcher_static.a libgomp.so.1 libLLVMOffload.so libompd.so libomptarget.so nvptx64-nvidia-cuda From 178389135e9847893645fcfebe3d3e5643cb2597 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 13 Dec 2025 09:43:35 -0800 Subject: [PATCH 48/48] try also adding libompdevice per vendor --- src/bootstrap/src/core/build_steps/compile.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index f4b8f71cd797d..d918e4ec84a4c 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2219,7 +2219,7 @@ impl Step for Assemble { if builder.config.llvm_offload && !builder.config.dry_run() { debug!("`llvm_offload` requested"); let offload_install = builder.ensure(llvm::OmpOffload { target: build_compiler.host }); - if let Some(llvm_config) = builder.llvm_config(builder.config.host_target) { + if let Some(_llvm_config) = builder.llvm_config(builder.config.host_target) { let src_dir = offload_install.join("lib"); let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); let target_libdir = @@ -2251,6 +2251,18 @@ impl Step for Assemble { // The last one is slightly more tricky, since we have the same file twice, in two // subfolders for amdgcn and nvptx64. We'll likely find two more in the future, once // Intel and Spir-V support lands in offload. + let gpu_tgts = ["amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + let device = format!("libompdevice.a"); + for tgt in gpu_tgts { + let dst_tgt_dir = libdir.join(&tgt); + let target_tgt_dst_dir = target_libdir.join(&tgt); + t!(fs::create_dir_all(&dst_tgt_dir)); + t!(fs::create_dir_all(&target_tgt_dst_dir)); + let dst_tgt_lib = dst_tgt_dir.join(&device); + let target_tgt_dst_lib = target_tgt_dst_dir.join(&device); + builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); + builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); + } //[\u@\h:\W]$ ls ../offload-outdir/lib // amdgcn-amd-amdhsa libarcher.so libgomp.so libiomp5.so libLLVMOffload.so.21.1 libomp.so libomptarget.so.21.1 // cmake libarcher_static.a libgomp.so.1 libLLVMOffload.so libompd.so libomptarget.so nvptx64-nvidia-cuda