Merge remote-tracking branch 'upstream/main' into proc_macro_group

Author: Keith-Cancel

compiler/rustc/Cargo.toml

@@ -28,16 +28,17 @@ wasi = "=0.14.2"
 
 
 [dependencies.tikv-jemalloc-sys]
-version = "0.6.0"
+version = "0.6.1"
 optional = true
-features = ['unprefixed_malloc_on_supported_platforms']
+features = ['override_allocator_on_supported_platforms']
 
 [features]
 # tidy-alphabetical-start
 check_only = ['rustc_driver_impl/check_only']
 jemalloc = ['dep:tikv-jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 llvm_enzyme = ['rustc_driver_impl/llvm_enzyme']
+llvm_offload = ['rustc_driver_impl/llvm_offload']
 max_level_info = ['rustc_driver_impl/max_level_info']
 rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
 # tidy-alphabetical-end

compiler/rustc/src/main.rs

@@ -7,26 +7,25 @@
 // distribution. The obvious way to do this is with the `#[global_allocator]`
 // mechanism. However, for complicated reasons (see
 // https://github.com/rust-lang/rust/pull/81782#issuecomment-784438001 for some
-// details) that mechanism doesn't work here. Also, we must use a consistent
-// allocator across the rustc <-> llvm boundary, and `#[global_allocator]`
-// wouldn't provide that.
+// details) that mechanism doesn't work here. Also, we'd like to use a
+// consistent allocator across the rustc <-> llvm boundary, and
+// `#[global_allocator]` wouldn't provide that.
 //
-// Instead, we use a lower-level mechanism. rustc is linked with jemalloc in a
-// way such that jemalloc's implementation of `malloc`, `free`, etc., override
-// the libc allocator's implementation. This means that Rust's `System`
-// allocator, which calls `libc::malloc()` et al., is actually calling into
-// jemalloc.
+// Instead, we use a lower-level mechanism, namely the
+// `"override_allocator_on_supported_platforms"` Cargo feature of jemalloc-sys.
+//
+// This makes jemalloc-sys override the libc/system allocator's implementation
+// of `malloc`, `free`, etc.. This means that Rust's `System` allocator, which
+// calls `libc::malloc()` et al., is actually calling into jemalloc.
 //
 // A consequence of not using `GlobalAlloc` (and the `tikv-jemallocator` crate
 // provides an impl of that trait, which is called `Jemalloc`) is that we
 // cannot use the sized deallocation APIs (`sdallocx`) that jemalloc provides.
 // It's unclear how much performance is lost because of this.
 //
-// As for the symbol overrides in `main` below: we're pulling in a static copy
-// of jemalloc. We need to actually reference its symbols for it to get linked.
-// The two crates we link to here, `std` and `rustc_driver`, are both dynamic
-// libraries. So we must reference jemalloc symbols one way or another, because
-// this file is the only object code in the rustc executable.
+// NOTE: Even though Cargo passes `--extern` with `tikv_jemalloc_sys`, we still need to `use` the
+// crate for the compiler to see the `#[used]`, see https://github.com/rust-lang/rust/issues/64402.
+// This is similarly required if we used a crate with `#[global_allocator]`.
 //
 // NOTE: if you are reading this comment because you want to set a custom `global_allocator` for
 // benchmarking, consider using the benchmarks in the `rustc-perf` collector suite instead:
@@ -36,43 +35,9 @@
 // to compare their performance, see
 // https://github.com/rust-lang/rust/commit/b90cfc887c31c3e7a9e6d462e2464db1fe506175#diff-43914724af6e464c1da2171e4a9b6c7e607d5bc1203fa95c0ab85be4122605ef
 // for an example of how to do so.
+#[cfg(feature = "jemalloc")]
+use tikv_jemalloc_sys as _;
 
 fn main() {
-    // See the comment at the top of this file for an explanation of this.
-    #[cfg(feature = "jemalloc")]
-    {
-        use std::os::raw::{c_int, c_void};
-
-        use tikv_jemalloc_sys as jemalloc_sys;
-
-        #[used]
-        static _F1: unsafe extern "C" fn(usize, usize) -> *mut c_void = jemalloc_sys::calloc;
-        #[used]
-        static _F2: unsafe extern "C" fn(*mut *mut c_void, usize, usize) -> c_int =
-            jemalloc_sys::posix_memalign;
-        #[used]
-        static _F3: unsafe extern "C" fn(usize, usize) -> *mut c_void = jemalloc_sys::aligned_alloc;
-        #[used]
-        static _F4: unsafe extern "C" fn(usize) -> *mut c_void = jemalloc_sys::malloc;
-        #[used]
-        static _F5: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = jemalloc_sys::realloc;
-        #[used]
-        static _F6: unsafe extern "C" fn(*mut c_void) = jemalloc_sys::free;
-
-        // On OSX, jemalloc doesn't directly override malloc/free, but instead
-        // registers itself with the allocator's zone APIs in a ctor. However,
-        // the linker doesn't seem to consider ctors as "used" when statically
-        // linking, so we need to explicitly depend on the function.
-        #[cfg(target_os = "macos")]
-        {
-            unsafe extern "C" {
-                fn _rjem_je_zone_register();
-            }
-
-            #[used]
-            static _F7: unsafe extern "C" fn() = _rjem_je_zone_register;
-        }
-    }
-
     rustc_driver::main()
 }

compiler/rustc_codegen_llvm/Cargo.toml

@@ -47,5 +47,6 @@ tracing = "0.1"
 # tidy-alphabetical-start
 check_only = ["rustc_llvm/check_only"]
 llvm_enzyme = []
+llvm_offload = []
 # tidy-alphabetical-end
 

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -771,6 +771,13 @@ pub(crate) unsafe fn llvm_optimize(
             llvm_plugins.len(),
         )
     };
+
+    if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Enable) {
+        unsafe {
+            llvm::LLVMRustBundleImages(module.module_llvm.llmod(), module.module_llvm.tm.raw());
+        }
+    }
+
     result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
 }
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1718,6 +1718,37 @@ unsafe extern "C" {
     ) -> &'a Value;
 }
 
+#[cfg(feature = "llvm_offload")]
+pub(crate) use self::Offload::*;
+
+#[cfg(feature = "llvm_offload")]
+mod Offload {
+    use super::*;
+    unsafe extern "C" {
+        /// Processes the module and writes it in an offload compatible way into a "host.out" file.
+        pub(crate) fn LLVMRustBundleImages<'a>(M: &'a Module, TM: &'a TargetMachine) -> bool;
+        pub(crate) fn LLVMRustOffloadMapper<'a>(OldFn: &'a Value, NewFn: &'a Value);
+    }
+}
+
+#[cfg(not(feature = "llvm_offload"))]
+pub(crate) use self::Offload_fallback::*;
+
+#[cfg(not(feature = "llvm_offload"))]
+mod Offload_fallback {
+    use super::*;
+    /// Processes the module and writes it in an offload compatible way into a "host.out" file.
+    /// Marked as unsafe to match the real offload wrapper which is unsafe due to FFI.
+    #[allow(unused_unsafe)]
+    pub(crate) unsafe fn LLVMRustBundleImages<'a>(_M: &'a Module, _TM: &'a TargetMachine) -> bool {
+        unimplemented!("This rustc version was not built with LLVM Offload support!");
+    }
+    #[allow(unused_unsafe)]
+    pub(crate) unsafe fn LLVMRustOffloadMapper<'a>(_OldFn: &'a Value, _NewFn: &'a Value) {
+        unimplemented!("This rustc version was not built with LLVM Offload support!");
+    }
+}
+
 // FFI bindings for `DIBuilder` functions in the LLVM-C API.
 // Try to keep these in the same order as in `llvm/include/llvm-c/DebugInfo.h`.
 //
@@ -2025,7 +2056,6 @@ unsafe extern "C" {
     ) -> &Attribute;
 
     // Operations on functions
-    pub(crate) fn LLVMRustOffloadMapper<'a>(Fn: &'a Value, Fn: &'a Value);
     pub(crate) fn LLVMRustGetOrInsertFunction<'a>(
         M: &'a Module,
         Name: *const c_char,

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -24,6 +24,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) {
         match *rvalue {
             mir::Rvalue::Use(ref operand) => {
+                if let mir::Operand::Constant(const_op) = operand {
+                    let val = self.eval_mir_constant(&const_op);
+                    if val.all_bytes_uninit(self.cx.tcx()) {
+                        return;
+                    }
+                }
                 let cg_operand = self.codegen_operand(bx, operand);
                 // Crucially, we do *not* use `OperandValue::Ref` for types with
                 // `BackendRepr::Scalar | BackendRepr::ScalarPair`. This ensures we match the MIR

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -522,6 +522,10 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
     pub(super) fn op(&self) -> &Operand<Prov> {
         &self.op
     }
+
+    pub fn is_immediate_uninit(&self) -> bool {
+        matches!(self.op, Operand::Immediate(Immediate::Uninit))
+    }
 }
 
 impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {

compiler/rustc_driver_impl/Cargo.toml

@@ -75,6 +75,7 @@ ctrlc = "3.4.4"
 check_only = ['rustc_interface/check_only']
 llvm = ['rustc_interface/llvm']
 llvm_enzyme = ['rustc_interface/llvm_enzyme']
+llvm_offload = ['rustc_interface/llvm_offload']
 max_level_info = ['rustc_log/max_level_info']
 rustc_randomized_layouts = [
     'rustc_index/rustc_randomized_layouts',

compiler/rustc_errors/src/lib.rs

@@ -348,7 +348,7 @@ impl CodeSuggestion {
             hi_opt: Option<&Loc>,
         ) -> usize {
             let mut line_count = 0;
-            // Convert CharPos to Usize, as CharPose is character offset
+            // Convert `CharPos` to `usize`, as `CharPos` is character offset
             // Extract low index and high index
             let (lo, hi_opt) = (lo.col.to_usize(), hi_opt.map(|hi| hi.col.to_usize()));
             if let Some(line) = line_opt {

compiler/rustc_interface/Cargo.toml

@@ -59,4 +59,5 @@ rustc_abi = { path = "../rustc_abi" }
 check_only = ['rustc_codegen_llvm?/check_only']
 llvm = ['dep:rustc_codegen_llvm']
 llvm_enzyme = ['rustc_builtin_macros/llvm_enzyme', 'rustc_codegen_llvm/llvm_enzyme']
+llvm_offload = ['rustc_codegen_llvm/llvm_offload']
 # tidy-alphabetical-end