guaranteed tail calls: support indirect arguments

folkertdev · folkertdev · commit 33942c64ad5d · 2026-01-15T20:24:37.000+01:00
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -1140,19 +1140,76 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (args, None)
         };
 
+        // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+        //
+        // Normally an indirect argument with `on_stack: false` would be passed as a pointer into
+        // the caller's stack frame. For tail calls, that would be unsound, because the caller's
+        // stack frame is overwritten by the callee's stack frame.
+        //
+        // Therefore we store the argument for the callee in the corresponding caller's slot.
+        // Because guaranteed tail calls demand that the caller's signature matches the callee's,
+        // the corresponding slot has the correct type.
+        //
+        // To handle cases like the one below, the tail call arguments must first be copied to a
+        // temporary, and only then copied to the caller's argument slots.
+        //
+        // ```
+        // // A struct big enough that it is not passed via registers.
+        // pub struct Big([u64; 4]);
+        //
+        // fn swapper(a: Big, b: Big) -> (Big, Big) {
+        //     become swapper_helper(b, a);
+        // }
+        // ```
+        let mut caller_indirect_arg_places = Vec::new();
+        if kind == CallKind::Tail {
+            let mut temporaries = vec![None; first_args.len()];
+            caller_indirect_arg_places = vec![None; first_args.len()];
+
+            // First copy the arguments of this call to temporary stack allocations.
+            for (i, arg) in first_args.iter().enumerate() {
+                if !matches!(fn_abi.args[i].mode, PassMode::Indirect { on_stack: false, .. }) {
+                    continue;
+                }
+
+                let mut op = self.codegen_operand(bx, &arg.node);
+
+                let tmp = PlaceRef::alloca(bx, op.layout);
+                bx.lifetime_start(tmp.val.llval, tmp.layout.size);
+                op.store_with_annotation(bx, tmp);
+
+                op.val = Ref(tmp.val);
+                temporaries[i] = Some(tmp);
+            }
+
+            // Then actually copy them into the corresponding argument place of our caller.
+            for (i, opt_tmp) in temporaries.into_iter().enumerate() {
+                let Some(tmp) = opt_tmp else { continue };
+
+                let local = self.mir.args_iter().nth(i).unwrap();
+
+                match &self.locals[local] {
+                    LocalRef::Place(arg) => {
+                        caller_indirect_arg_places[i] = Some(Ref(arg.val));
+                        bx.typed_place_copy(arg.val, tmp.val, fn_abi.args[i].layout);
+                    }
+                    LocalRef::Operand(arg) => {
+                        caller_indirect_arg_places[i] = Some(arg.val);
+                        arg.store_with_annotation(bx, tmp);
+                    }
+                    LocalRef::UnsizedPlace(_) => bug!("unsized types are not supported"),
+                    LocalRef::PendingOperand => bug!("argument local should not be pending"),
+                };
+
+                bx.lifetime_end(tmp.val.llval, tmp.layout.size);
+            }
+        }
+
         // When generating arguments we sometimes introduce temporary allocations with lifetime
         // that extend for the duration of a call. Keep track of those allocations and their sizes
         // to generate `lifetime_end` when the call returns.
         let mut lifetime_ends_after_call: Vec<(Bx::Value, Size)> = Vec::new();
         'make_args: for (i, arg) in first_args.iter().enumerate() {
-            if kind == CallKind::Tail && matches!(fn_abi.args[i].mode, PassMode::Indirect { .. }) {
-                // FIXME: https://github.com/rust-lang/rust/pull/144232#discussion_r2218543841
-                span_bug!(
-                    fn_span,
-                    "arguments using PassMode::Indirect are currently not supported for tail calls"
-                );
-            }
-
             let mut op = self.codegen_operand(bx, &arg.node);
 
             if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, instance.map(|i| i.def)) {
@@ -1203,18 +1260,47 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            // The callee needs to own the argument memory if we pass it
-            // by-ref, so make a local copy of non-immediate constants.
-            match (&arg.node, op.val) {
-                (&mir::Operand::Copy(_), Ref(PlaceValue { llextra: None, .. }))
-                | (&mir::Operand::Constant(_), Ref(PlaceValue { llextra: None, .. })) => {
-                    let tmp = PlaceRef::alloca(bx, op.layout);
-                    bx.lifetime_start(tmp.val.llval, tmp.layout.size);
-                    op.store_with_annotation(bx, tmp);
-                    op.val = Ref(tmp.val);
-                    lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
+            match kind {
+                CallKind::Tail => {
+                    match fn_abi.args[i].mode {
+                        PassMode::Indirect { on_stack: false, .. } => {
+                            let Some(dst_val) = caller_indirect_arg_places[i] else {
+                                span_bug!(fn_span, "missing caller place for tail call arg {i}");
+                            };
+
+                            // The argument has been stored in the caller's argument place above.
+                            // Now forward that place to the callee.
+                            op.val = dst_val;
+                        }
+                        PassMode::Indirect { on_stack: true, .. } => {
+                            // FIXME: some LLVM backends (notably x86) do not correctly pass byval
+                            // arguments to tail calls (as of LLVM 21). See also:
+                            //
+                            // - https://github.com/rust-lang/rust/pull/144232#discussion_r2218543841
+                            // - https://github.com/rust-lang/rust/issues/144855
+                            span_bug!(
+                                fn_span,
+                                "arguments using PassMode::Indirect {{ on_stack: true, .. }} are currently not supported for tail calls"
+                            )
+                        }
+                        _ => (),
+                    }
+                }
+                CallKind::Normal => {
+                    // The callee needs to own the argument memory if we pass it
+                    // by-ref, so make a local copy of non-immediate constants.
+                    match (&arg.node, op.val) {
+                        (&mir::Operand::Copy(_), Ref(PlaceValue { llextra: None, .. }))
+                        | (&mir::Operand::Constant(_), Ref(PlaceValue { llextra: None, .. })) => {
+                            let tmp = PlaceRef::alloca(bx, op.layout);
+                            bx.lifetime_start(tmp.val.llval, tmp.layout.size);
+                            op.store_with_annotation(bx, tmp);
+                            op.val = Ref(tmp.val);
+                            lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
+                        }
+                        _ => {}
+                    }
                 }
-                _ => {}
             }
 
             self.codegen_argument(
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -121,17 +121,29 @@ impl<'tcx> Visitor<'tcx> for DeduceParamAttrs {
         // `f` passes. Note that function arguments are the only situation in which this problem can
         // arise: every other use of `move` in MIR doesn't actually write to the value it moves
         // from.
-        if let TerminatorKind::Call { ref args, .. } = terminator.kind {
-            for arg in args {
-                if let Operand::Move(place) = arg.node
-                    && !place.is_indirect_first_projection()
-                    && let Some(i) = self.as_param(place.local)
-                {
-                    self.usage[i] |= UsageSummary::MUTATE;
-                    self.usage[i] |= UsageSummary::CAPTURE;
+        match terminator.kind {
+            TerminatorKind::Call { ref args, .. } => {
+                for arg in args {
+                    if let Operand::Move(place) = arg.node
+                        && !place.is_indirect_first_projection()
+                        && let Some(i) = self.as_param(place.local)
+                    {
+                        self.usage[i] |= UsageSummary::MUTATE;
+                        self.usage[i] |= UsageSummary::CAPTURE;
+                    }
                 }
             }
-        };
+
+            // Like a call, but more conservative because the backend may introduce writes to an
+            // argument if the argument is passed as `PassMode::Indirect { on_stack: false, ... }`.
+            TerminatorKind::TailCall { .. } => {
+                for usage in self.usage.iter_mut() {
+                    *usage |= UsageSummary::MUTATE;
+                    *usage |= UsageSummary::CAPTURE;
+                }
+            }
+            _ => {}
+        }
 
         self.super_terminator(terminator, location);
     }
diff --git a/tests/assembly-llvm/tail-call-indirect.rs b/tests/assembly-llvm/tail-call-indirect.rs
@@ -0,0 +1,75 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//@ needs-llvm-components: x86
+//@ compile-flags: --target=x86_64-unknown-linux-gnu
+//@ compile-flags: -Copt-level=3 -C llvm-args=-x86-asm-syntax=intel
+
+#![feature(no_core, explicit_tail_calls)]
+#![expect(incomplete_features)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+//
+// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
+// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// frame is overwritten by the callee's stack frame.
+//
+// The solution is to write the argument into the caller's argument place (stored somewhere further
+// up the stack), and forward that place.
+
+extern crate minicore;
+use minicore::*;
+
+#[repr(C)]
+struct S {
+    x: u64,
+    y: u64,
+    z: u64,
+}
+
+unsafe extern "C" {
+    safe fn force_usage(_: u64, _: u64, _: u64) -> u64;
+}
+
+// CHECK-LABEL: callee:
+// CHECK-NEXT: .cfi_startproc
+//
+// CHECK-NEXT: mov rax, qword ptr [rdi]
+// CHECK-NEXT: mov rsi, qword ptr [rdi + 8]
+// CHECK-NEXT: mov rdx, qword ptr [rdi + 16]
+// CHECK-NEXT: mov rdi, rax
+//
+// CHECK-NEXT: jmp qword ptr [rip + force_usage@GOTPCREL]
+#[inline(never)]
+#[unsafe(no_mangle)]
+fn callee(s: S) -> u64 {
+    force_usage(s.x, s.y, s.z)
+}
+
+// CHECK-LABEL: caller1:
+// CHECK-NEXT: .cfi_startproc
+//
+// Just forward the argument:
+//
+// CHECK-NEXT: jmp qword ptr [rip + callee@GOTPCREL]
+#[unsafe(no_mangle)]
+fn caller1(s: S) -> u64 {
+    become callee(s);
+}
+
+// CHECK-LABEL: caller2:
+// CHECK-NEXT: .cfi_startproc
+//
+// Construct the S value directly into the argument slot:
+//
+// CHECK-NEXT: mov qword ptr [rdi], 1
+// CHECK-NEXT: mov qword ptr [rdi + 8], 2
+// CHECK-NEXT: mov qword ptr [rdi + 16], 3
+//
+// CHECK-NEXT: jmp qword ptr [rip + callee@GOTPCREL]
+#[unsafe(no_mangle)]
+fn caller2(_: S) -> u64 {
+    let s = S { x: 1, y: 2, z: 3 };
+    become callee(s);
+}
diff --git a/tests/crashes/144293-indirect-ops-llvm.rs b/tests/crashes/144293-indirect-ops-llvm.rs
diff --git a/tests/ui/explicit-tail-calls/indirect.rs b/tests/ui/explicit-tail-calls/indirect.rs
@@ -0,0 +1,65 @@
+//@ run-pass
+//@ ignore-backends: gcc
+#![feature(explicit_tail_calls)]
+#![expect(incomplete_features)]
+
+// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+//
+// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
+// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// frame is overwritten by the callee's stack frame.
+//
+// The solution is to write the argument into the caller's argument place (stored somewhere further
+// up the stack), and forward that place.
+
+// A struct big enough that it is not passed via registers, so that the rust calling convention uses
+// `Indirect { on_stack: false, .. }`.
+#[repr(C)]
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Big([u64; 4]);
+
+#[inline(never)]
+fn update_in_caller(y: Big) -> u64 {
+    #[inline(never)]
+    fn helper(x: Big) -> u64 {
+        x.0.iter().sum()
+    }
+
+    let x = Big([y.0[0], 2, 3, 4]);
+
+    // `x` is actually stored in `y`'s space.
+    become helper(x)
+}
+
+#[inline(never)]
+fn swapper<T>(a: T, b: T) -> (T, T) {
+    #[inline(never)]
+    fn helper<T>(a: T, b: T) -> (T, T) {
+        (a, b)
+    }
+
+    become helper(b, a)
+}
+
+#[inline(never)]
+fn swapper_derived(a: Big, _: (u64, u64), b: Big, _: (u64, u64)) -> ((u64, u64), (u64, u64)) {
+    #[inline(never)]
+    fn helper(_: Big, x: (u64, u64), _: Big, y: (u64, u64)) -> ((u64, u64), (u64, u64)) {
+        (x, y)
+    }
+
+    // Read the values at various points in the swapping process, testing that they have the correct
+    // value at every point.
+    become helper(b, (a.0[0], b.0[0]), a, (a.0[0], b.0[0]));
+}
+
+fn main() {
+    assert_eq!(update_in_caller(Big::default()), 0 + 2 + 3 + 4);
+
+    assert_eq!(swapper(u8::MIN, u8::MAX), (u8::MAX, u8::MIN));
+    // i128 uses `PassMode::Indirect { on_stack: false, .. }` on x86_64 MSVC.
+    assert_eq!(swapper(i128::MIN, i128::MAX), (i128::MAX, i128::MIN));
+    assert_eq!(swapper(Big([1; 4]), Big([2; 4])), (Big([2; 4]), Big([1; 4])));
+
+    assert_eq!(swapper_derived(Big([1; 4]), (0, 0), Big([2; 4]), (0, 0)), ((1, 2), (1, 2)));
+}
