diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs index e92c4353fb70..b74eb8a291ad 100644 --- a/crates/cranelift/src/alias_region.rs +++ b/crates/cranelift/src/alias_region.rs @@ -16,6 +16,7 @@ enum VmType { VMStoreContext, VMMemoryDefinition, VMTableDefinition, + VMDeferredThread, } /// A key that uniquely identifies an alias region across an entire compilation. @@ -105,6 +106,7 @@ impl AliasRegionKey { const GC_HEAP_KIND: u32 = Self::new_kind(0b1000); const VM_MEMORY_DEFINITION_KIND: u32 = Self::new_kind(0b1001); const VM_TABLE_DEFINITION_KIND: u32 = Self::new_kind(0b1010); + const VM_DEFERRED_THREAD_KIND: u32 = Self::new_kind(0b1011); /// Encode this key into a raw `u32` suitable for use as an /// `AliasRegionData::user_id`. @@ -117,6 +119,7 @@ impl AliasRegionKey { VmType::VMStoreContext => Self::VM_STORE_CONTEXT_KIND, VmType::VMMemoryDefinition => Self::VM_MEMORY_DEFINITION_KIND, VmType::VMTableDefinition => Self::VM_TABLE_DEFINITION_KIND, + VmType::VMDeferredThread => Self::VM_DEFERRED_THREAD_KIND, }; kind | (offset & Self::OFFSET_MASK) } @@ -1092,6 +1095,44 @@ where ) } + /// Load the `VMStoreContext::current_thread` field (the JIT-visible + /// deferred-thread pointer; see `VMLazyThread`). + pub fn vmstore_context_current_thread( + &mut self, + cursor: &mut FuncCursor<'_>, + vmstore_ctx: ir::Value, + ) -> ir::Value { + self.vmstore_context_load( + cursor, + self.pointer_type, + ir::MemFlagsData::trusted(), + vmstore_ctx, + self.offsets + .get_ptr_size() + .vmstore_context_current_thread() + .into(), + ) + } + + /// Store the `VMStoreContext::current_thread` field. + pub fn store_vmstore_context_current_thread( + &mut self, + cursor: &mut FuncCursor<'_>, + vmstore_ctx: ir::Value, + new_thread: ir::Value, + ) { + self.vmstore_context_store( + cursor, + ir::MemFlagsData::trusted(), + vmstore_ctx, + self.offsets + .get_ptr_size() + .vmstore_context_current_thread() + .into(), + new_thread, + ) + } + /// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`). /// /// The caller supplies the base flags because whether the base pointer is @@ -1326,6 +1367,194 @@ where } } +/// `VMDeferredThread`-related methods. +impl AliasRegions +where + Offsets: GetPtrSize, +{ + fn vmdeferred_thread_region( + &mut self, + func: &mut ir::Function, + offset: u32, + ) -> ir::AliasRegion { + self.region( + func, + AliasRegionKey::Vm { + ty: VmType::VMDeferredThread, + offset, + }, + ) + } + + fn vmdeferred_thread_load( + &mut self, + cursor: &mut FuncCursor<'_>, + ty: ir::Type, + base_flags: ir::MemFlagsData, + vmdeferred_thread_ptr: ir::Value, + offset: u32, + ) -> ir::Value { + let region = self.vmdeferred_thread_region(cursor.func, offset); + cursor.ins().load( + ty, + base_flags.with_alias_region(Some(region)), + vmdeferred_thread_ptr, + i32::try_from(offset).unwrap(), + ) + } + + fn vmdeferred_thread_store( + &mut self, + cursor: &mut FuncCursor<'_>, + base_flags: ir::MemFlagsData, + vmdeferred_thread_ptr: ir::Value, + offset: u32, + val: ir::Value, + ) { + let region = self.vmdeferred_thread_region(cursor.func, offset); + cursor.ins().store( + base_flags.with_alias_region(Some(region)), + val, + vmdeferred_thread_ptr, + i32::try_from(offset).unwrap(), + ); + } + + /// Load `VMDeferredThread::parent` (the current thread this frame replaced). + pub fn vmdeferred_thread_parent( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + ) -> ir::Value { + self.vmdeferred_thread_load( + cursor, + self.pointer_type, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_parent() + .into(), + ) + } + + /// Store `VMDeferredThread::parent`. + pub fn store_vmdeferred_thread_parent( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + parent: ir::Value, + ) { + self.vmdeferred_thread_store( + cursor, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_parent() + .into(), + parent, + ) + } + + /// Store `VMDeferredThread::caller_instance`. + pub fn store_vmdeferred_thread_caller_instance( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + caller_instance: ir::Value, + ) { + self.vmdeferred_thread_store( + cursor, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_caller_instance() + .into(), + caller_instance, + ) + } + + /// Store `VMDeferredThread::callee_async`. + pub fn store_vmdeferred_thread_callee_async( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + callee_async: ir::Value, + ) { + self.vmdeferred_thread_store( + cursor, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_callee_async() + .into(), + callee_async, + ) + } + + /// Store `VMDeferredThread::callee_instance`. + pub fn store_vmdeferred_thread_callee_instance( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + callee_instance: ir::Value, + ) { + self.vmdeferred_thread_store( + cursor, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_callee_instance() + .into(), + callee_instance, + ) + } + + /// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}` + /// slot). + pub fn vmdeferred_thread_saved_context( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + i: u8, + ) -> ir::Value { + self.vmdeferred_thread_load( + cursor, + ir::types::I32, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_saved_context(i) + .into(), + ) + } + + /// Store `VMDeferredThread::saved_context[i]`. + pub fn store_vmdeferred_thread_saved_context( + &mut self, + cursor: &mut FuncCursor<'_>, + vmdeferred_thread_ptr: ir::Value, + i: u8, + val: ir::Value, + ) { + self.vmdeferred_thread_store( + cursor, + ir::MemFlagsData::trusted(), + vmdeferred_thread_ptr, + self.offsets + .get_ptr_size() + .vmdeferred_thread_saved_context(i) + .into(), + val, + ) + } +} + /// `VMMemoryDefinition`-related methods. /// /// The `base` and `current_length` fields are reached either directly through diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index 793cd20664de..b22c9999a7f3 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -627,6 +627,10 @@ impl wasmtime_environ::Compiler for Compiler { unreachable!() } + // Never names a compiled function; only marks an adapter import for + // inline lowering in `known_imported_functions`. + FuncKey::FactInlineIntrinsic(..) => unreachable!(), + FuncKey::ModuleStartup(abi, module) => { let translation = translation.unwrap(); let ty = match translation.module.startup { diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9134d1fcc7f6..23c3acbc82af 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -35,13 +35,14 @@ use wasmparser::{ use wasmtime_core::math::f64_cvt_to_int_bounds; use wasmtime_environ::{ BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, - DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, + DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, FactInlineIntrinsic, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module, - ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize, - RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements, - TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, - WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType, + ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, NUM_COMPONENT_CONTEXT_SLOTS, + PassiveElemIndex, PtrSize, RuntimeDataIndex, Table, TableIndex, TableInitialValue, + TableSegment, TableSegmentElements, TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, + WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, + WasmStorageType, WasmValType, }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; @@ -166,6 +167,12 @@ pub struct FuncEnvironment<'module_environment> { /// Heaps implementing WebAssembly linear memories. heaps: PrimaryMap, + /// When translating a fused sync-to-sync adapter, the explicit stack slot + /// holding the `VMDeferredThread` pushed by the inline `enter-sync-call` + /// lowering, shared with the matching inline `exit-sync-call`. `None` + /// outside that window. + fact_sync_call_slot: Option, + /// Caches of signatures for builtin functions. builtin_functions: BuiltinFunctions, @@ -276,6 +283,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { gc_heap: None, heaps: PrimaryMap::default(), + fact_sync_call_slot: None, builtin_functions, offsets, tunables, @@ -1905,6 +1913,36 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { Ok(self.direct_call_inst(callee, &real_call_args)) } + // The guest-to-guest sync fast path: these adapter intrinsics are + // lowered inline rather than called, but only when concurrency + // support is enabled (the deferred thread state only exists then) + // and this isn't a tail call (the deferred frame must outlive the + // call). Otherwise fall back to the indirect call, which is also + // the out-of-line slow path the inline `exit` branches to. + Some(FuncKey::FactInlineIntrinsic(intrinsic)) => { + if self.env.tunables.concurrency_support { + debug_assert!(!self.tail); + match intrinsic { + FactInlineIntrinsic::EnterSyncCall => { + return Ok(self.lower_fact_enter_sync_call(&real_call_args)); + } + FactInlineIntrinsic::ExitSyncCall => { + return Ok(self.lower_fact_exit_sync_call( + callee_index, + sig_ref, + &real_call_args, + )); + } + } + } + let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call( + &mut self.builder.cursor(), + vmctx, + callee_index, + ); + Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args)) + } + Some(key) => panic!("unexpected kind of known-import function: {key:?}"), // Unknown import function or this module is instantiated many times @@ -1935,6 +1973,174 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest } + /// Inline lowering of a FACT adapter's `enter-sync-call` intrinsic: push a + /// `VMDeferredThread` onto an explicit stack slot and publish it as the + /// store's current thread, deferring the heavyweight task bookkeeping the + /// `enter_sync_call` libcall would otherwise do eagerly. + /// + /// `real_call_args` is `[callee_vmctx, caller_vmctx, caller_instance, + /// callee_async, callee_instance]`. + fn lower_fact_enter_sync_call(&mut self, real_call_args: &[ir::Value]) -> CallRets { + let ptr_ty = self.env.pointer_type(); + let ptr = self.env.offsets.ptr; + let flags = ir::MemFlagsData::trusted(); + + // Allocate the on-stack `VMDeferredThread`. + let size = u32::from(ptr.size_of_vmdeferred_thread()); + let align_shift = u8::try_from(ptr.size().trailing_zeros()).unwrap(); + let slot = self + .builder + .func + .create_sized_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + size, + align_shift, + )); + let slot_addr = self.builder.ins().stack_addr(ptr_ty, slot, 0); + + let vmstore = self.env.get_vmstore_context_ptr(self.builder); + + // Link the previous current thread in as this frame's parent. + let parent = self + .env + .alias_regions + .vmstore_context_current_thread(&mut self.builder.cursor(), vmstore); + self.env.alias_regions.store_vmdeferred_thread_parent( + &mut self.builder.cursor(), + slot_addr, + parent, + ); + + // Record the deferred `enter_sync_call` arguments. + self.env + .alias_regions + .store_vmdeferred_thread_caller_instance( + &mut self.builder.cursor(), + slot_addr, + real_call_args[2], + ); + self.env.alias_regions.store_vmdeferred_thread_callee_async( + &mut self.builder.cursor(), + slot_addr, + real_call_args[3], + ); + self.env + .alias_regions + .store_vmdeferred_thread_callee_instance( + &mut self.builder.cursor(), + slot_addr, + real_call_args[4], + ); + + // Save the caller's context slots into the frame and reset the live + // values to 0 for the freshly-entered (deferred) thread. + for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() { + let off = i32::from(ptr.vmstore_context_component_context_slot(i)); + let saved = self.builder.ins().load(ir::types::I32, flags, vmstore, off); + self.env + .alias_regions + .store_vmdeferred_thread_saved_context( + &mut self.builder.cursor(), + slot_addr, + i, + saved, + ); + let zero = self.builder.ins().iconst(ir::types::I32, 0); + self.builder.ins().store(flags, zero, vmstore, off); + } + + // Publish the deferred thread as the store's current thread. + self.env.alias_regions.store_vmstore_context_current_thread( + &mut self.builder.cursor(), + vmstore, + slot_addr, + ); + + debug_assert!(self.env.fact_sync_call_slot.is_none()); + self.env.fact_sync_call_slot = Some(slot); + CallRets::new() + } + + /// Inline lowering of a FACT adapter's `exit-sync-call` intrinsic, the + /// counterpart to `lower_fact_enter_sync_call`. If our deferred thread is + /// still current (nothing forced it) we pop it and restore the caller's + /// context inline; otherwise we fall back to the out-of-line + /// `exit_sync_call` libcall. + fn lower_fact_exit_sync_call( + &mut self, + callee_index: FuncIndex, + sig_ref: ir::SigRef, + real_call_args: &[ir::Value], + ) -> CallRets { + let ptr_ty = self.env.pointer_type(); + let ptr = self.env.offsets.ptr; + let flags = ir::MemFlagsData::trusted(); + + let slot = self + .env + .fact_sync_call_slot + .take() + .expect("inline exit-sync-call without a matching enter-sync-call"); + let slot_addr = self.builder.ins().stack_addr(ptr_ty, slot, 0); + let vmstore = self.env.get_vmstore_context_ptr(self.builder); + let cur = self + .env + .alias_regions + .vmstore_context_current_thread(&mut self.builder.cursor(), vmstore); + let is_fast = self.builder.ins().icmp(IntCC::Equal, cur, slot_addr); + + let fast_block = self.builder.create_block(); + let slow_block = self.builder.create_block(); + let cont_block = self.builder.create_block(); + self.builder + .ins() + .brif(is_fast, fast_block, &[], slow_block, &[]); + self.builder.seal_block(fast_block); + self.builder.seal_block(slow_block); + + // Fast path: pop the deferred thread and restore the caller's context. + self.builder.switch_to_block(fast_block); + let parent = self + .env + .alias_regions + .vmdeferred_thread_parent(&mut self.builder.cursor(), slot_addr); + self.env.alias_regions.store_vmstore_context_current_thread( + &mut self.builder.cursor(), + vmstore, + parent, + ); + for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() { + let saved = self.env.alias_regions.vmdeferred_thread_saved_context( + &mut self.builder.cursor(), + slot_addr, + i, + ); + self.builder.ins().store( + flags, + saved, + vmstore, + i32::from(ptr.vmstore_context_component_context_slot(i)), + ); + } + self.builder.ins().jump(cont_block, &[]); + + // Slow path: the thread was promoted to a real one, so do the + // equivalent out-of-line teardown via the `exit_sync_call` libcall. + self.builder.switch_to_block(slow_block); + let vmctx = self.env.vmctx_val(&mut self.builder.cursor()); + let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call( + &mut self.builder.cursor(), + vmctx, + callee_index, + ); + self.indirect_call_inst(sig_ref, func_addr, real_call_args); + self.builder.ins().jump(cont_block, &[]); + + self.builder.seal_block(cont_block); + self.builder.switch_to_block(cont_block); + CallRets::new() + } + /// Do a Wasm-level indirect call through the given funcref table. pub fn indirect_call( mut self, diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 542181e55fd6..483847ef1a44 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -37,6 +37,38 @@ pub struct ModuleEnvironment<'a, 'data> { tunables: &'a Tunables, } +/// Identifies a FACT adapter-module import that the compiler lowers inline when +/// translating the adapter function. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum FactInlineIntrinsic { + /// `enter-sync-call`: push a deferred component-model thread inline. + EnterSyncCall, + /// `exit-sync-call`: pop the deferred thread inline on the fast path, or + /// fall back to the out-of-line `exit-sync-call` libcall when the thread + /// was promoted. + ExitSyncCall, +} + +impl FactInlineIntrinsic { + /// Get this intrinsic's raw `u32` representation (for `FuncKey` packing). + pub fn into_raw(self) -> u32 { + self as u32 + } + + /// Reconstruct from a raw representation produced by [`Self::into_raw`]. + /// + /// Panics on invalid input. + pub fn from_raw(raw: u32) -> Self { + match raw { + x if x == Self::EnterSyncCall.into_raw() => Self::EnterSyncCall, + x if x == Self::ExitSyncCall.into_raw() => Self::ExitSyncCall, + _ => panic!( + "invalid raw representation passed to `FactInlineIntrinsic::from_raw`: {raw}" + ), + } + } +} + /// The result of translating via `ModuleEnvironment`. /// /// Function bodies are not yet translated, and data initializers have not yet @@ -68,7 +100,8 @@ pub struct ModuleTranslation<'data> { /// imports table into direct calls, when possible. /// /// When filled in, this only ever contains - /// `FuncKey::DefinedWasmFunction(..)`s and `FuncKey::Intrinsic(..)`s. + /// `FuncKey::DefinedWasmFunction(..)`s, `FuncKey::Intrinsic(..)`s, and + /// `FuncKey::FactInlineIntrinsic`s. pub known_imported_functions: SecondaryMap>, /// A list of type signatures which are considered exported from this diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs index 447ec4faa853..928129dd0eff 100644 --- a/crates/environ/src/component/translate.rs +++ b/crates/environ/src/component/translate.rs @@ -3,9 +3,9 @@ use crate::component::dfg::AbstractInstantiations; use crate::component::*; use crate::prelude::*; use crate::{ - EngineOrModuleTypeIndex, EntityIndex, FuncKey, ModuleEnvironment, ModuleInternedTypeIndex, - ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex, Tunables, TypeConvert, - WasmHeapType, WasmResult, WasmValType, + EngineOrModuleTypeIndex, EntityIndex, FactInlineIntrinsic, FuncKey, ModuleEnvironment, + ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex, + Tunables, TypeConvert, WasmHeapType, WasmResult, WasmValType, }; use core::str::FromStr; use cranelift_entity::SecondaryMap; @@ -623,7 +623,20 @@ impl<'a, 'data> Translator<'a, 'data> { // turn them into direct calls even though we probably // wouldn't ever inline them, but it just doesn't seem worth // the effort. - CoreDef::Trampoline(_) => continue, + // + // That said, a couple of adapter trampolines are lowered + // inline during translation. We record these here so + // `FuncEnvironment` recognizes them. All other trampolines + // remain indirect calls. + CoreDef::Trampoline(index) => match translation.trampolines[*index] { + Trampoline::EnterSyncCall => { + FuncKey::FactInlineIntrinsic(FactInlineIntrinsic::EnterSyncCall) + } + Trampoline::ExitSyncCall => { + FuncKey::FactInlineIntrinsic(FactInlineIntrinsic::ExitSyncCall) + } + _ => continue, + }, // This import is a compile-time builtin intrinsic, we // should inline its implementation during function diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs index d046a998b76c..c078947352fa 100644 --- a/crates/environ/src/fact/trampoline.rs +++ b/crates/environ/src/fact/trampoline.rs @@ -789,9 +789,6 @@ impl<'a, 'b> Compiler<'a, 'b> { }; // Push a task onto the current task stack. - // - // FIXME: Apply the optimizations described in #12311. - self.instruction(I32Const( i32::try_from(adapter.lower.instance.as_u32()).unwrap(), )); @@ -870,8 +867,6 @@ impl<'a, 'b> Compiler<'a, 'b> { // "you forgot to drop borrows" trap shows up and additionally the // lowering below may call realloc which is in the context of the // caller's task, not the callee. - // - // FIXME: Apply the optimizations described in #12311. if self.emit_resource_call || self.module.tunables.concurrency_support { let exit_sync_call = self.module.import_exit_sync_call(); self.instruction(Call(exit_sync_call.as_u32())); diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs index 66bc7c189a8d..cdcca609baef 100644 --- a/crates/environ/src/key.rs +++ b/crates/environ/src/key.rs @@ -50,6 +50,15 @@ pub enum FuncKeyKind { /// Initialization function for a module, such as initializing "complicated" /// globals and passive element segments. ModuleStartup = FuncKey::new_kind(0b1001), + + /// A FACT adapter intrinsic that the compiler lowers inline rather than + /// calling (the guest-to-guest sync fast path; see [`FuncKey`]). + /// + /// This never names a compiled function; it only ever appears as a value in + /// `ModuleTranslation::known_imported_functions` to mark an adapter import + /// for inline lowering. + #[cfg(feature = "component-model")] + FactInlineIntrinsic = FuncKey::new_kind(0b1010), } impl From for u32 { @@ -85,6 +94,8 @@ impl FuncKeyKind { x if x == Self::ResourceDropTrampoline.into() => Self::ResourceDropTrampoline, #[cfg(feature = "component-model")] x if x == Self::UnsafeIntrinsic.into() => Self::UnsafeIntrinsic, + #[cfg(feature = "component-model")] + x if x == Self::FactInlineIntrinsic.into() => Self::FactInlineIntrinsic, _ => panic!("invalid raw value passed to `FuncKind::from_raw`: {raw}"), } @@ -158,6 +169,12 @@ impl FuncKeyNamespace { let _ = Abi::from_raw(raw & FuncKey::MODULE_MASK); Self(raw) } + + #[cfg(feature = "component-model")] + FuncKeyKind::FactInlineIntrinsic => { + assert_eq!(raw & FuncKey::MODULE_MASK, 0); + Self(raw) + } } } @@ -267,6 +284,15 @@ pub enum FuncKey { /// This function has the `Abi` specified and will initialize the module /// specified. ModuleStartup(Abi, StaticModuleIndex), + + /// A FACT adapter intrinsic that the compiler lowers inline rather than + /// calling. + /// + /// This never names a compiled function. It only ever appears as a value in + /// `ModuleTranslation::known_imported_functions`, where it tells + /// `FuncEnvironment` to lower the corresponding adapter import inline. + #[cfg(feature = "component-model")] + FactInlineIntrinsic(crate::FactInlineIntrinsic), } impl Ord for FuncKey { @@ -362,6 +388,13 @@ impl FuncKey { let index = abi.into_raw(); (namespace, index) } + + #[cfg(feature = "component-model")] + FuncKey::FactInlineIntrinsic(intrinsic) => { + let namespace = FuncKeyKind::FactInlineIntrinsic.into_raw(); + let index = intrinsic.into_raw(); + (namespace, index) + } }; (FuncKeyNamespace(namespace), FuncKeyIndex(index)) } @@ -397,6 +430,8 @@ impl FuncKey { #[cfg(feature = "component-model")] FuncKey::UnsafeIntrinsic(abi, _) => abi, FuncKey::ModuleStartup(abi, _) => abi, + #[cfg(feature = "component-model")] + FuncKey::FactInlineIntrinsic(_) => Abi::Wasm, } } @@ -489,6 +524,12 @@ impl FuncKey { let abi = Abi::from_raw(b); Self::ModuleStartup(abi, module) } + + #[cfg(feature = "component-model")] + FuncKeyKind::FactInlineIntrinsic => { + assert_eq!(a & Self::MODULE_MASK, 0); + Self::FactInlineIntrinsic(crate::FactInlineIntrinsic::from_raw(b)) + } } } @@ -593,7 +634,8 @@ impl FuncKey { #[cfg(feature = "component-model")] Self::ComponentTrampoline(..) | Self::ResourceDropTrampoline - | Self::UnsafeIntrinsic(..) => true, + | Self::UnsafeIntrinsic(..) + | Self::FactInlineIntrinsic(..) => true, } } } diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index c3173b3819e5..368de01b17e3 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -555,10 +555,13 @@ impl CompiledFunctionsTable { | FuncKeyKind::PatchableToBuiltinTrampoline | FuncKeyKind::ModuleStartup => false, + // `FactInlineIntrinsic` never names a compiled function, so it is + // never classified here; group it with the other intrinsics. #[cfg(feature = "component-model")] FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline - | FuncKeyKind::UnsafeIntrinsic => true, + | FuncKeyKind::UnsafeIntrinsic + | FuncKeyKind::FactInlineIntrinsic => true, } } @@ -575,7 +578,8 @@ impl CompiledFunctionsTable { #[cfg(feature = "component-model")] FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline - | FuncKeyKind::UnsafeIntrinsic => false, + | FuncKeyKind::UnsafeIntrinsic + | FuncKeyKind::FactInlineIntrinsic => false, } } } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index df1cbc516fc5..244ab92e3910 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -279,6 +279,46 @@ pub trait PtrSize { base + i * slot_size } + /// Return the offset of the `current_thread` field of `VMStoreContext`. + fn vmstore_context_current_thread(&self) -> u8 { + self.vmstore_context_component_context_slot(0) + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4 + } + + // Offsets within `VMDeferredThread` + + /// Offset of `VMDeferredThread::parent`. + fn vmdeferred_thread_parent(&self) -> u8 { + 0 + } + + /// Offset of `VMDeferredThread::caller_instance`. + fn vmdeferred_thread_caller_instance(&self) -> u8 { + self.size() + } + + /// Offset of `VMDeferredThread::callee_async`. + fn vmdeferred_thread_callee_async(&self) -> u8 { + self.vmdeferred_thread_caller_instance() + 4 + } + + /// Offset of `VMDeferredThread::callee_instance`. + fn vmdeferred_thread_callee_instance(&self) -> u8 { + self.vmdeferred_thread_callee_async() + 4 + } + + /// Offset of `VMDeferredThread::saved_context[i]`. + fn vmdeferred_thread_saved_context(&self, i: u8) -> u8 { + assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS); + self.vmdeferred_thread_callee_instance() + 4 + i * 4 + } + + /// Return the size of `VMDeferredThread`, rounded up to pointer alignment. + fn size_of_vmdeferred_thread(&self) -> u8 { + let unaligned = + self.vmdeferred_thread_callee_instance() + 4 + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4; + align(u32::from(unaligned), u32::from(self.size())) as u8 + } + // Offsets within `VMMemoryDefinition` /// The offset of the `base` field. diff --git a/crates/wasi-tls/src/p3/host.rs b/crates/wasi-tls/src/p3/host.rs index 857072116e55..66c084f415e2 100644 --- a/crates/wasi-tls/src/p3/host.rs +++ b/crates/wasi-tls/src/p3/host.rs @@ -101,7 +101,7 @@ impl bindings::tls::client::HostConnectorWithStore for WasiTls { .map_err(|e| Error::from(e)); _ = send_result_tx.send(combined_result); Ok(()) - })); + }))?; let result = ResultProducer::new(getter, send_result_rx); Ok(( @@ -167,7 +167,7 @@ impl bindings::tls::client::HostConnectorWithStore for WasiTls { .map_err(|e| Error::from(e)); _ = recv_result_tx.send(combined_result); Ok(()) - })); + }))?; let result = ResultProducer::new(getter, recv_result_rx); Ok(( diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 077f4753600c..ca77e6484b40 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -969,6 +969,10 @@ fn is_inlining_function(key: FuncKey) -> bool { | FuncKey::ModuleStartup(..) => false, FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false, + // Never names a compiled function (it only marks an adapter import for + // inline lowering), so it never participates in inlining itself. + FuncKey::FactInlineIntrinsic(..) => false, + FuncKey::PulleyHostCall(_) => { unreachable!("we don't compile artifacts for Pulley host calls") } diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rs index 5ec4de15bb15..eb42270cc7df 100644 --- a/crates/wasmtime/src/runtime/component/concurrent.rs +++ b/crates/wasmtime/src/runtime/component/concurrent.rs @@ -61,7 +61,7 @@ use crate::hash_set::HashSet; use crate::prelude::*; use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken}; use crate::vm::component::{CallContext, ComponentInstance, InstanceState}; -use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore}; +use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore}; use crate::{ AsContext, AsContextMut, FuncType, Result, StoreContext, StoreContextMut, ValRaw, ValType, bail, }; @@ -235,7 +235,7 @@ where /// Spawn a background task. /// /// See [`Accessor::spawn`] for details. - pub fn spawn(&mut self, task: impl AccessorTask) -> JoinHandle + pub fn spawn(&mut self, task: impl AccessorTask) -> Result where T: 'static, { @@ -511,7 +511,7 @@ where /// Panics if called within a closure provided to the [`Accessor::with`] /// function. This can only be called outside an active invocation of /// [`Accessor::with`]. - pub fn spawn(&self, task: impl AccessorTask) -> JoinHandle + pub fn spawn(&self, task: impl AccessorTask) -> Result where T: 'static, { @@ -564,7 +564,7 @@ where pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> { self.with(|mut access| { let store = access.as_context_mut().0; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut_without_forcing_current_thread(); if state.interesting_tasks == 0 { Poll::Ready(()) } else { @@ -717,7 +717,7 @@ impl GuestCall { /// instance to be called has backpressure enabled fn is_ready(&self, store: &mut StoreOpaque) -> Result { let instance = store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(self.thread.task)? .instance; let state = store.instance_state(instance).concurrent_state(); @@ -795,8 +795,7 @@ pub(crate) fn poll_and_block( store: &mut dyn VMStore, future: impl Future> + Send + 'static, ) -> Result { - let state = store.concurrent_state_mut(); - let task = state.current_host_thread()?; + let task = store.current_host_thread()?; // Wrap the future in a closure which will take care of stashing the result // in `GuestTask::result` and resuming this fiber when the host task @@ -804,7 +803,7 @@ pub(crate) fn poll_and_block( let mut future = Box::pin(async move { let result = future.await?; tls::get(move |store| { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let host_state = &mut state.get_mut(task)?.state; assert!(matches!(host_state, HostTaskState::CalleeStarted)); *host_state = HostTaskState::CalleeFinished(Box::new(result)); @@ -838,7 +837,7 @@ pub(crate) fn poll_and_block( // then use `GuestThread::sync_call_set` to wait for the task to // complete, suspending the current fiber until it does so. Poll::Pending => { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; state.push_future(future); let caller = state.get_mut(task)?.caller; @@ -854,12 +853,12 @@ pub(crate) fn poll_and_block( // Remove the `task` from the `sync_call_set` to ensure that when // this function returns and the task is deleted that there are no // more lingering references to this host task. - Waitable::Host(task).join(store.concurrent_state_mut(), None)?; + Waitable::Host(task).join(store.concurrent_state_mut()?, None)?; } } // Retrieve and return the result. - let host_state = &mut store.concurrent_state_mut().get_mut(task)?.state; + let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state; match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) { HostTaskState::CalleeFinished(result) => Ok(match result.downcast() { Ok(result) => *result, @@ -880,7 +879,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> { Some(pair) => pair, None => bail_bug!("delivering non-present event"), }; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let task = state.get_mut(call.thread.task)?; let runtime_instance = task.instance; let handle = waitable.map(|(_, v)| v).unwrap_or(0); @@ -899,7 +898,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> { store.enter_instance(runtime_instance); let Some(callback) = store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(call.thread.task)? .callback .take() @@ -910,7 +909,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> { let code = callback(store, event, handle)?; store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(call.thread.task)? .callback = Some(callback); @@ -965,7 +964,7 @@ impl Store { } /// Convenience wrapper for [`StoreContextMut::spawn`]. - pub fn spawn(&mut self, task: impl AccessorTask>) -> JoinHandle + pub fn spawn(&mut self, task: impl AccessorTask>) -> Result where T: 'static, { @@ -991,7 +990,7 @@ impl StoreContextMut<'_, T> { .store_data_mut() .components .assert_instance_states_empty(); - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut().unwrap(); assert!( state.table.get_mut().is_empty(), "non-empty table: {:?}", @@ -999,7 +998,7 @@ impl StoreContextMut<'_, T> { ); assert!(state.high_priority.is_empty()); assert!(state.low_priority.is_empty()); - assert!(state.current_thread.is_none()); + assert!(state.unforced_current_thread.is_none()); assert!(state.futures_mut().unwrap().is_empty()); assert!(state.global_error_context_ref_counts.is_empty()); } @@ -1012,6 +1011,7 @@ impl StoreContextMut<'_, T> { pub fn concurrent_state_table_size(&mut self) -> usize { self.0 .concurrent_state_mut() + .unwrap() .table .get_mut() .iter_mut() @@ -1027,7 +1027,7 @@ impl StoreContextMut<'_, T> { /// for this instance is run. /// /// The returned [`JoinHandle`] may be used to cancel the task. - pub fn spawn(mut self, task: impl AccessorTask) -> JoinHandle + pub fn spawn(mut self, task: impl AccessorTask) -> Result where T: 'static, { @@ -1041,7 +1041,7 @@ impl StoreContextMut<'_, T> { self, accessor: Accessor, task: impl AccessorTask, - ) -> JoinHandle + ) -> Result where T: 'static, D: HasData + ?Sized, @@ -1051,9 +1051,9 @@ impl StoreContextMut<'_, T> { // iteration. let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await }); self.0 - .concurrent_state_mut() + .concurrent_state_mut()? .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) })); - handle + Ok(handle) } /// Run the specified closure `fun` to completion as part of this store's @@ -1225,7 +1225,12 @@ impl StoreContextMut<'_, T> { impl<'a, T> Drop for Reset<'a, T> { fn drop(&mut self) { if let Some(futures) = self.futures.take() { - *self.store.0.concurrent_state_mut().futures.get_mut() = Some(futures); + *self + .store + .0 + .concurrent_state_mut_already_forced_current_thread() + .futures + .get_mut() = Some(futures); } } } @@ -1234,7 +1239,7 @@ impl StoreContextMut<'_, T> { // Take `ConcurrentState::futures` out of the store so we can poll // it while also safely giving any of the futures inside access to // `self`. - let futures = self.0.concurrent_state_mut().futures.get_mut().take(); + let futures = self.0.concurrent_state_mut()?.futures.get_mut().take(); let mut reset = Reset { store: self.as_context_mut(), futures, @@ -1277,7 +1282,7 @@ impl StoreContextMut<'_, T> { // Next, collect the next batch of work items to process, if // any. This will be either all of the high-priority work // items, or if there are none, a single low-priority work item. - let state = reset.store.0.concurrent_state_mut(); + let state = reset.store.0.concurrent_state_mut()?; let mut ready = mem::take(&mut state.high_priority); let mut low_priority = false; if ready.is_empty() { @@ -1428,7 +1433,7 @@ impl StoreContextMut<'_, T> { match item { WorkItem::PushFuture(future) => { self.0 - .concurrent_state_mut() + .concurrent_state_mut()? .futures_mut()? .push(future.into_inner()); } @@ -1437,7 +1442,7 @@ impl StoreContextMut<'_, T> { } WorkItem::ResumeThread(_, thread) => { if let GuestThreadState::Ready { fiber, .. } = mem::replace( - &mut self.0.concurrent_state_mut().get_mut(thread.thread)?.state, + &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state, GuestThreadState::Running, ) { self.0.resume_fiber(fiber).await?; @@ -1449,7 +1454,7 @@ impl StoreContextMut<'_, T> { if call.is_ready(self.0)? { self.run_on_worker(WorkerItem::GuestCall(call)).await?; } else { - let state = self.0.concurrent_state_mut(); + let state = self.0.concurrent_state_mut()?; let task = state.get_mut(call.thread.task)?; if !task.starting_sent { task.starting_sent = true; @@ -1484,12 +1489,12 @@ impl StoreContextMut<'_, T> { where T: Send, { - let worker = if let Some(fiber) = self.0.concurrent_state_mut().worker.take() { + let worker = if let Some(fiber) = self.0.concurrent_state_mut()?.worker.take() { fiber } else { fiber::make_fiber(self.0, move |store| { loop { - let Some(item) = store.concurrent_state_mut().worker_item.take() else { + let Some(item) = store.concurrent_state_mut()?.worker_item.take() else { bail_bug!("worker_item not present when resuming fiber") }; match item { @@ -1502,7 +1507,7 @@ impl StoreContextMut<'_, T> { })? }; - let worker_item = &mut self.0.concurrent_state_mut().worker_item; + let worker_item = &mut self.0.concurrent_state_mut()?.worker_item; assert!(worker_item.is_none()); *worker_item = Some(item); @@ -1538,10 +1543,10 @@ impl StoreContextMut<'_, T> { /// Tasks are yielded "youngest first" where the first item in the iterator /// is the current task, and the last item in the iterator is the original /// call. - pub fn async_call_stack(&mut self) -> impl Iterator { - let state = self.0.concurrent_state_mut(); - let mut cur = Some(state.current_thread); - core::iter::from_fn(move || { + pub fn async_call_stack(&mut self) -> Result> { + let mut cur = Some(self.0.current_thread()?); + let state = self.0.concurrent_state_mut()?; + Ok(core::iter::from_fn(move || { while let Some(t) = cur { cur = state.parent(t); if let Some(thread) = t.guest() { @@ -1550,11 +1555,117 @@ impl StoreContextMut<'_, T> { } None - }) + })) } } impl StoreOpaque { + /// Returns the currently-running thread, promoting any deferred lazy thread + /// into a fully-materialized `CurrentThread`. + pub(crate) fn current_thread(&mut self) -> Result { + // Without concurrency support there is nothing to force. + if !self.concurrency_support() { + return Ok(CurrentThread::None); + } + + // If the JIT-visible current thread isn't a deferred thread then + // `ConcurrentState` is already up to date. + if !self.vm_store_context().current_thread.is_deferred() { + return Ok(self + .concurrent_state_mut_already_forced_current_thread() + .unforced_current_thread); + } + + // The component instance whose adapters pushed the deferred frames; all + // frames in a guest-to-guest, sync-to-sync call chain of fused adapters + // live within a single `wasmtime::component::Instance` (because + // cross-`wasmtime::component::Instance` calls don't go through fused + // adapters), and guest code only ever runs as a guest thread, so the + // chain's base thread is already materialized in `ConcurrentState` and + // we can get the `ComponentInstanceId` shared by the whole chain from + // here. + let state = self.concurrent_state_mut_without_forcing_current_thread(); + let id = match state.unforced_current_thread.guest().copied() { + Some(thread) => state.get_mut(thread.task)?.instance.instance, + None => bail_bug!("deferred component-model thread with non-guest base"), + }; + + // Collect the deferred frames pushed inline by fused adapters, walking + // the `parent` chain from innermost to the base. + let mut frames = Vec::new(); + let mut cur = self.vm_store_context().current_thread; + while let Some(ptr) = cur.as_deferred() { + // SAFETY: `ptr` points at a `VMDeferredThread` living in a fused + // adapter's stack frame that is suspended below us on the stack + // (mid-call, waiting for this nested call to return), so the + // referent is still valid and exclusively ours to read. + let deferred = unsafe { &*ptr }; + frames.push(( + deferred.callee_async != 0, + deferred.callee_instance, + deferred.saved_context, + )); + cur = deferred.parent; + } + + // Mark the current thread forced *before* replaying so that any + // reentrant `force_current_thread` call short-circuits via the + // non-deferred path above. + self.vm_store_context_mut().current_thread = VMLazyThread::forced(); + + // Save the current context, as we need to overwrite it while replaying + // below. + let current_context = self.vm_store_context().component_context; + + // Replay the deferred `enter_guest_sync_call`s outermost-first so that + // the resulting `ConcurrentState` matches what the non-deferred path + // would have otherwise produced. + for (callee_async, callee_instance, saved_context) in frames.into_iter().rev() { + // Restore the caller's context slots so that we save the correct + // values into the caller's thread, exactly as the non-deferred path + // would have on entry. + self.vm_store_context_mut().component_context = saved_context; + let callee = RuntimeInstance { + instance: id, + index: RuntimeComponentInstanceIndex::from_u32(callee_instance), + }; + self.enter_guest_sync_call(None, callee_async, callee)?; + } + + // Replaying done; restore the current context. + self.vm_store_context_mut().component_context = current_context; + + Ok(self + .concurrent_state_mut_without_forcing_current_thread() + .unforced_current_thread) + } + + fn current_guest_thread(&mut self) -> Result { + match self.current_thread()?.guest() { + Some(id) => Ok(*id), + None => bail_bug!("current thread is not a guest thread"), + } + } + + fn current_host_thread(&mut self) -> Result> { + match self.current_thread()?.host() { + Some(id) => Ok(id), + None => bail_bug!("current thread is not a host thread"), + } + } + + /// Returns whether there's a pending cancellation on the current guest thread, + /// consuming the event if so. + fn take_pending_cancellation(&mut self) -> Result { + let thread = self.current_guest_thread()?; + let task = self.concurrent_state_mut()?.get_mut(thread.task)?; + if let Some(Event::Cancelled) = task.event { + task.event.take(); + return Ok(true); + } + Ok(false) + } + /// Push a `GuestTask` onto the task stack for either a sync-to-sync, /// guest-to-guest call or a sync host-to-guest call. /// @@ -1572,8 +1683,8 @@ impl StoreOpaque { return self.enter_call_not_concurrent(); } - let state = self.concurrent_state_mut(); - let thread = state.current_thread; + let thread = self.current_thread()?; + let state = self.concurrent_state_mut()?; let instance = if let Some(thread) = thread.guest() { Some(state.get_mut(thread.task)?.instance) } else { @@ -1624,7 +1735,7 @@ impl StoreOpaque { Some(t) => *t, None => bail_bug!("expected task when exiting"), }; - let task = self.concurrent_state_mut().get_mut(thread.task)?; + let task = self.concurrent_state_mut()?.get_mut(thread.task)?; let instance = task.instance; let caller = match &task.caller { &Caller::Guest { thread } => thread.into(), @@ -1652,8 +1763,8 @@ impl StoreOpaque { self.enter_call_not_concurrent()?; return Ok(None); } - let state = self.concurrent_state_mut(); - let caller = state.current_guest_thread()?; + let caller = self.current_guest_thread()?; + let state = self.concurrent_state_mut()?; let task = state.push(HostTask::new(caller, HostTaskState::CalleeStarted))?; log::trace!("new host task {task:?}"); self.set_thread(task)?; @@ -1669,8 +1780,8 @@ impl StoreOpaque { if !self.concurrency_support() { return Ok(()); } - let task = self.concurrent_state_mut().current_host_thread()?; - let caller = self.concurrent_state_mut().get_mut(task)?.caller; + let task = self.current_host_thread()?; + let caller = self.concurrent_state_mut()?.get_mut(task)?.caller; self.set_thread(caller)?; Ok(()) } @@ -1685,7 +1796,7 @@ impl StoreOpaque { match task { Some(task) => { log::trace!("delete host task {task:?}"); - self.concurrent_state_mut().delete(task)?; + self.concurrent_state_mut()?.delete(task)?; } None => { self.exit_call_not_concurrent(); @@ -1708,8 +1819,8 @@ impl StoreOpaque { if !self.concurrency_support() { return Ok(true); } - let state = self.concurrent_state_mut(); - let mut cur = Some(state.current_thread); + let mut cur = Some(self.current_thread()?); + let state = self.concurrent_state_mut()?; while let Some(t) = cur { if let Some(thread) = t.guest() { let task = state.get_mut(thread.task)?; @@ -1742,8 +1853,8 @@ impl StoreOpaque { /// needed too. fn set_thread(&mut self, thread: impl Into) -> Result { let thread = thread.into(); - let state = self.concurrent_state_mut(); - let old_thread = mem::replace(&mut state.current_thread, thread); + let state = self.concurrent_state_mut()?; + let old_thread = mem::replace(&mut state.unforced_current_thread, thread); // First thing to do after swapping threads is updating the context // slots for this thread within the store. This restores the behavior of @@ -1754,7 +1865,7 @@ impl StoreOpaque { // this may be forgotten. if let Some(old_thread) = old_thread.guest() { let old_context = self.vm_store_context().component_context; - self.concurrent_state_mut() + self.concurrent_state_mut()? .get_mut(old_thread.thread)? .context = old_context; } @@ -1762,7 +1873,7 @@ impl StoreOpaque { self.vm_store_context_mut().component_context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS]; } if let Some(thread) = thread.guest() { - let thread = self.concurrent_state_mut().get_mut(thread.thread)?; + let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?; let context = thread.context; if cfg!(debug_assertions) { thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS]; @@ -1777,7 +1888,7 @@ impl StoreOpaque { // // Additionally if we're switching to a new thread, set its component // instance's `task_may_block` according to where it left off. - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; if let Some(old_thread) = old_thread.guest() { let instance = state.get_mut(old_thread.task)?.instance.instance; self.component_instance_mut(instance) @@ -1788,16 +1899,23 @@ impl StoreOpaque { self.set_task_may_block()?; } + // Keep the JIT-visible current-thread pointer in sync. + self.vm_store_context_mut().current_thread = if thread.is_none() { + VMLazyThread::none() + } else { + VMLazyThread::forced() + }; + Ok(old_thread) } /// Set the global variable representing whether the current task may block /// prior to entering Wasm code. fn set_task_may_block(&mut self) -> Result<()> { - let state = self.concurrent_state_mut(); - let guest_thread = state.current_guest_thread()?; + let guest_thread = self.current_guest_thread()?; + let state = self.concurrent_state_mut()?; let instance = state.get_mut(guest_thread.task)?.instance.instance; - let may_block = self.concurrent_state_mut().may_block(guest_thread.task)?; + let may_block = self.concurrent_state_mut()?.may_block(guest_thread.task)?; self.component_instance_mut(instance) .set_task_may_block(may_block); Ok(()) @@ -1807,8 +1925,8 @@ impl StoreOpaque { if !self.concurrency_support() { return Ok(()); } - let state = self.concurrent_state_mut(); - let task = state.current_guest_thread()?.task; + let task = self.current_guest_thread()?.task; + let state = self.concurrent_state_mut()?; let instance = state.get_mut(task)?.instance.instance; let task_may_block = self.component_instance(instance).get_task_may_block(); @@ -1850,7 +1968,7 @@ impl StoreOpaque { { let call = GuestCall { thread, kind }; if call.is_ready(self)? { - self.concurrent_state_mut() + self.concurrent_state_mut()? .push_high_priority(WorkItem::GuestCall(instance.index, call)); } else { self.instance_state(instance) @@ -1886,14 +2004,14 @@ impl StoreOpaque { /// Resume the specified fiber, giving it exclusive access to the specified /// store. async fn resume_fiber(&mut self, fiber: StoreFiber<'static>) -> Result<()> { - let old_thread = self.concurrent_state_mut().current_thread; + let old_thread = self.current_thread()?; log::trace!("resume_fiber: save current thread {old_thread:?}"); let fiber = fiber::resolve_or_release(self, fiber).await?; self.set_thread(old_thread)?; - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; if let Some(ot) = old_thread.guest() { state.get_mut(ot.thread)?.state = GuestThreadState::Running; @@ -1962,7 +2080,7 @@ impl StoreOpaque { }; let old_guest_thread = if task.is_some() { - self.concurrent_state_mut().current_thread + self.current_thread()? } else { CurrentThread::None }; @@ -1987,12 +2105,12 @@ impl StoreOpaque { } ) || old_guest_thread .guest() - .map(|thread| self.concurrent_state_mut().may_block(thread.task)) + .map(|thread| self.concurrent_state_mut()?.may_block(thread.task)) .transpose()? .unwrap_or(true) ); - let suspend_reason = &mut self.concurrent_state_mut().suspend_reason; + let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason; assert!(suspend_reason.is_none()); *suspend_reason = Some(reason); @@ -2006,13 +2124,13 @@ impl StoreOpaque { } fn wait_for_event(&mut self, waitable: Waitable) -> Result<()> { - let state = self.concurrent_state_mut(); + let caller = self.current_guest_thread()?; + let state = self.concurrent_state_mut()?; if waitable.common(state)?.set.is_some() { bail!(Trap::WaitableSyncAndAsync); } - let caller = state.current_guest_thread()?; let set = state.get_mut(caller.thread)?.sync_call_set; waitable.join(state, Some(set))?; self.suspend(SuspendReason::Waiting { @@ -2020,7 +2138,7 @@ impl StoreOpaque { thread: caller, skip_may_block_check: false, })?; - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; waitable.join(state, None) } @@ -2051,7 +2169,7 @@ impl StoreOpaque { runtime_instance: RuntimeInstance, cleanup_task: CleanupTask, ) -> Result<()> { - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; let thread_data = state.get_mut(guest_thread.thread)?; let sync_call_set = thread_data.sync_call_set; if let Some(guest_id) = thread_data.instance_rep { @@ -2059,7 +2177,7 @@ impl StoreOpaque { .thread_handle_table() .guest_thread_remove(guest_id)?; } - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; // Clean up any pending subtasks in the sync_call_set for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) { @@ -2122,7 +2240,7 @@ impl StoreOpaque { caller_instance: RuntimeInstance, guest_task: TableId, ) -> Result<()> { - let concurrent_state = self.concurrent_state_mut(); + let concurrent_state = self.concurrent_state_mut()?; let task = concurrent_state.get_mut(guest_task)?; assert!(!task.already_lowered_parameters()); // The task is in a `starting` state, meaning it hasn't run at @@ -2173,7 +2291,7 @@ impl Instance { set: Option>, cancellable: bool, ) -> Result)>> { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let event = &mut state.get_mut(guest_task)?.event; if let Some(ev) = event @@ -2229,7 +2347,7 @@ impl Instance { log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})"); - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let get_set = |store: &mut StoreOpaque, handle| -> Result<_> { let set = store @@ -2243,7 +2361,7 @@ impl Instance { Ok(match code { callback_code::EXIT => { log::trace!("implicit thread {guest_thread:?} completed"); - let task = store.concurrent_state_mut().get_mut(guest_thread.task)?; + let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?; task.exited = true; task.callback = None; store.cleanup_thread( @@ -2289,7 +2407,7 @@ impl Instance { state.check_blocking_for(guest_thread.task)?; let set = get_set(store, set)?; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; if state.get_mut(guest_thread.task)?.event.is_some() || !state.get_mut(set)?.ready.is_empty() @@ -2381,10 +2499,10 @@ impl Instance { let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS]; store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(guest_thread.thread)? .state = GuestThreadState::Running; - let task = store.concurrent_state_mut().get_mut(guest_thread.task)?; + let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?; let lower = match task.lower_params.take() { Some(l) => l, None => bail_bug!("lower_params missing"), @@ -2427,7 +2545,7 @@ impl Instance { let callee_instance = store .0 - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(guest_thread.task)? .instance; @@ -2458,7 +2576,7 @@ impl Instance { store.exit_instance(callee_instance)?; store.set_thread(old_thread)?; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; if let Some(t) = old_thread.guest() { state.get_mut(t.thread)?.state = GuestThreadState::Running; } @@ -2509,7 +2627,7 @@ impl Instance { let lift = { store.exit_instance(callee_instance)?; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; if !state.get_mut(guest_thread.task)?.result.is_none() { bail_bug!("task has already produced a result"); } @@ -2551,7 +2669,7 @@ impl Instance { store.set_thread(old_thread)?; store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(guest_thread.task)? .exited = true; @@ -2563,7 +2681,7 @@ impl Instance { store .0 - .concurrent_state_mut() + .concurrent_state_mut()? .push_high_priority(WorkItem::GuestCall( callee_instance.index, GuestCall { @@ -2647,8 +2765,8 @@ impl Instance { let start = SendSyncPtr::new(start); let return_ = SendSyncPtr::new(return_); let token = StoreToken::new(store.as_context_mut()); - let state = store.0.concurrent_state_mut(); - let old_thread = state.current_guest_thread()?; + let old_thread = store.0.current_guest_thread()?; + let state = store.0.concurrent_state_mut()?; debug_assert_eq!( state.get_mut(old_thread.task)?.instance, @@ -2699,8 +2817,9 @@ impl Instance { )?; } dst.copy_from_slice(&src[..dst.len()]); - let state = store.0.concurrent_state_mut(); - Waitable::Guest(state.current_guest_thread()?.task).set_event( + let task = store.0.current_guest_thread()?.task; + let state = store.0.concurrent_state_mut()?; + Waitable::Guest(task).set_event( state, Some(Event::Subtask { status: Status::Started, @@ -2743,8 +2862,8 @@ impl Instance { // lifting/lowering has returned. store.0.set_thread(prev)?; - let state = store.0.concurrent_state_mut(); - let thread = state.current_guest_thread()?; + let thread = store.0.current_guest_thread()?; + let state = store.0.concurrent_state_mut()?; if sync_caller { state.get_mut(thread.task)?.sync_result = SyncResult::Produced( if let ResultInfo::Stack { result_count } = &result_info { @@ -2834,8 +2953,8 @@ impl Instance { ) -> Result { let token = StoreToken::new(store.as_context_mut()); let async_caller = storage.is_none(); - let state = store.0.concurrent_state_mut(); - let guest_thread = state.current_guest_thread()?; + let guest_thread = store.0.current_guest_thread()?; + let state = store.0.concurrent_state_mut()?; let callee_async = state.get_mut(guest_thread.task)?.async_function; let callee = SendSyncPtr::new(callee); let param_count = usize::try_from(param_count)?; @@ -2877,7 +2996,7 @@ impl Instance { )?; } - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; // Use the caller's `GuestThread::sync_call_set` to register interest in // the subtask... @@ -2917,7 +3036,7 @@ impl Instance { skip_may_block_check: async_caller || !callee_async, })?; - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; log::trace!("taking event for {:?}", guest_thread.task); let event = guest_waitable.take_event(state)?; @@ -2941,7 +3060,7 @@ impl Instance { .subtask_insert_guest(guest_thread.task.rep())?; store .0 - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(guest_thread.task)? .common .handle = Some(handle); @@ -2953,18 +3072,22 @@ impl Instance { } }; - guest_waitable.join(store.0.concurrent_state_mut(), old_set)?; + guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?; // Reset the current thread to point to the caller as it resumes control. store.0.set_thread(caller)?; - store.0.concurrent_state_mut().get_mut(caller.thread)?.state = GuestThreadState::Running; + store + .0 + .concurrent_state_mut()? + .get_mut(caller.thread)? + .state = GuestThreadState::Running; log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}"); if let Some(storage) = storage { // The caller used a sync-lowered import to call an async-lifted // export, in which case the result, if any, has been stashed in // `GuestTask::sync_result`. - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; let task = state.get_mut(guest_thread.task)?; if let Some(result) = task.sync_result.take()? { if let Some(result) = result { @@ -2999,8 +3122,8 @@ impl Instance { lower: impl FnOnce(StoreContextMut, Option) -> Result<()> + Send + 'static, ) -> Result> { let token = StoreToken::new(store.as_context_mut()); - let state = store.0.concurrent_state_mut(); - let task = state.current_host_thread()?; + let task = store.0.current_host_thread()?; + let state = store.0.concurrent_state_mut()?; // Create an abortable future which hooks calls to poll and manages call // context state for the future. @@ -3061,7 +3184,7 @@ impl Instance { }; lower(store.as_context_mut(), result)?; - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; match &mut state.get_mut(task)?.state { // The task is already flagged as finished because it was // cancelled. No need to transition further. @@ -3082,7 +3205,7 @@ impl Instance { // there are host embedder frames on the stack is unsound. tls::get(move |store| { store - .concurrent_state_mut() + .concurrent_state_mut()? .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new( on_complete, )))); @@ -3092,7 +3215,7 @@ impl Instance { // Make this task visible to the guest and then record what it // was made visible as. - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; state.push_future(future); let caller = state.get_mut(task)?.caller; let instance = state.get_mut(caller.task)?.instance; @@ -3101,7 +3224,7 @@ impl Instance { .instance_state(instance) .handle_table() .subtask_insert_host(task.rep())?; - store.0.concurrent_state_mut().get_mut(task)?.common.handle = Some(handle); + store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle); log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}"); // Restore the currently running thread to this host task's @@ -3120,8 +3243,8 @@ impl Instance { options: OptionsIndex, storage: &[ValRaw], ) -> Result<()> { - let state = store.concurrent_state_mut(); - let guest_thread = state.current_guest_thread()?; + let guest_thread = store.current_guest_thread()?; + let state = store.concurrent_state_mut()?; let lift = state .get_mut(guest_thread.task)? .lift_result @@ -3166,8 +3289,8 @@ impl Instance { /// Implements the `task.cancel` intrinsic. pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> { - let state = store.concurrent_state_mut(); - let guest_thread = state.current_guest_thread()?; + let guest_thread = store.current_guest_thread()?; + let state = store.concurrent_state_mut()?; let task = state.get_mut(guest_thread.task)?; if !task.cancel_sent { bail!(Trap::TaskCancelNotCancelled); @@ -3204,10 +3327,10 @@ impl Instance { status: Status, ) -> Result<()> { store - .component_resource_tables(Some(self)) + .component_resource_tables(Some(self))? .validate_scope_exit()?; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let task = state.get_mut(guest_task)?; if let Caller::Host { tx, .. } = &mut task.caller { @@ -3228,7 +3351,7 @@ impl Instance { store: &mut StoreOpaque, caller_instance: RuntimeComponentInstanceIndex, ) -> Result { - let set = store.concurrent_state_mut().push(WaitableSet::default())?; + let set = store.concurrent_state_mut()?.push(WaitableSet::default())?; let handle = store .instance_state(self.runtime_instance(caller_instance)) .handle_table() @@ -3255,7 +3378,7 @@ impl Instance { // set to avoid dropping any waiters in `WaitMode::Fiber(_)`, which // would panic. See `drop-waitable-set-with-waiters.wast` for details. if !store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(TableId::::new(rep))? .waiting .is_empty() @@ -3264,7 +3387,7 @@ impl Instance { } store - .concurrent_state_mut() + .concurrent_state_mut()? .delete(TableId::::new(rep))?; Ok(()) @@ -3289,7 +3412,7 @@ impl Instance { .handle_table() .waitable_set_rep(set_handle)?; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; if let Some(old) = waitable.common(state)?.set && state.get_mut(old)?.is_sync_call_set { @@ -3303,7 +3426,7 @@ impl Instance { "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})", ); - waitable.join(store.concurrent_state_mut(), set) + waitable.join(store.concurrent_state_mut()?, set) } /// Implements the `subtask.drop` intrinsic. @@ -3320,7 +3443,7 @@ impl Instance { .handle_table() .subtask_remove(task_id)?; - let concurrent_state = store.concurrent_state_mut(); + let concurrent_state = store.concurrent_state_mut()?; let (waitable, delete) = if is_host { let id = TableId::::new(rep); let task = concurrent_state.get_mut(id)?; @@ -3429,9 +3552,9 @@ impl Instance { /// Implements the `thread.index` intrinsic. pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result { - let thread_id = store.concurrent_state_mut().current_guest_thread()?.thread; + let thread_id = store.current_guest_thread()?.thread; match store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(thread_id)? .instance_rep { @@ -3483,7 +3606,7 @@ impl Instance { CleanupTask::Yes, )?; log::trace!("explicit thread {guest_thread:?} completed"); - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; if let Some(t) = old_thread.guest() { state.get_mut(t.thread)?.state = GuestThreadState::Running; } @@ -3493,8 +3616,8 @@ impl Instance { }, ); - let state = store.0.concurrent_state_mut(); - let current_thread = state.current_guest_thread()?; + let current_thread = store.0.current_guest_thread()?; + let state = store.0.concurrent_state_mut()?; let parent_task = current_thread.task; let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?; @@ -3516,7 +3639,7 @@ impl Instance { ) -> Result<()> { let thread_id = GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let guest_thread = QualifiedThreadId::qualify(state, thread_id)?; let thread = state.get_mut(guest_thread.thread)?; @@ -3533,20 +3656,20 @@ impl Instance { }, ); store - .concurrent_state_mut() + .concurrent_state_mut()? .push_work_item(guest_call, high_priority); } GuestThreadState::Suspended(fiber) => { log::trace!("resuming thread {thread_id:?} that was suspended"); store - .concurrent_state_mut() + .concurrent_state_mut()? .push_work_item(WorkItem::ResumeFiber(fiber), high_priority); } GuestThreadState::Ready { fiber, cancellable } if allow_ready => { log::trace!("resuming thread {thread_id:?} that was ready"); thread.state = GuestThreadState::Ready { fiber, cancellable }; store - .concurrent_state_mut() + .concurrent_state_mut()? .promote_thread_work_item(guest_thread); } other => { @@ -3568,7 +3691,7 @@ impl Instance { .thread_handle_table() .guest_thread_insert(thread_id.rep())?; store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(thread_id)? .instance_rep = Some(guest_id); Ok(guest_id) @@ -3584,9 +3707,9 @@ impl Instance { yielding: bool, to_thread: SuspensionTarget, ) -> Result { - let guest_thread = store.concurrent_state_mut().current_guest_thread()?; + let guest_thread = store.current_guest_thread()?; if to_thread.is_none() { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; if yielding { // This is a `thread.yield` call if !state.may_block(guest_thread.task)? { @@ -3606,7 +3729,7 @@ impl Instance { } // There could be a pending cancellation from a previous uncancellable wait - if cancellable && store.concurrent_state_mut().take_pending_cancellation()? { + if cancellable && store.take_pending_cancellation()? { return Ok(WaitResult::Cancelled); } @@ -3641,7 +3764,7 @@ impl Instance { store.suspend(reason)?; - if cancellable && store.concurrent_state_mut().take_pending_cancellation()? { + if cancellable && store.take_pending_cancellation()? { Ok(WaitResult::Cancelled) } else { Ok(WaitResult::Completed) @@ -3656,11 +3779,11 @@ impl Instance { check: WaitableCheck, params: WaitableCheckParams, ) -> Result { - let guest_thread = store.concurrent_state_mut().current_guest_thread()?; + let guest_thread = store.current_guest_thread()?; log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set); - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let task = state.get_mut(guest_thread.task)?; // If we're waiting, and there are no events immediately available, @@ -3762,7 +3885,7 @@ impl Instance { } else { Waitable::Guest(TableId::::new(rep)) }; - let concurrent_state = store.concurrent_state_mut(); + let concurrent_state = store.concurrent_state_mut()?; log::trace!("subtask_cancel {waitable:?} (handle {task_id})"); @@ -3842,7 +3965,7 @@ impl Instance { }; concurrent_state.push_high_priority(item); - let caller = concurrent_state.current_guest_thread()?; + let caller = store.current_guest_thread()?; store.suspend(SuspendReason::Yielding { thread: caller, cancellable: false, @@ -3857,8 +3980,8 @@ impl Instance { { // The thread is in a cancellable yield, so yield back // to it. - let caller = concurrent_state.current_guest_thread()?; concurrent_state.promote_thread_work_item(thread); + let caller = store.current_guest_thread()?; store.suspend(SuspendReason::Yielding { thread: caller, cancellable: false, @@ -3871,7 +3994,7 @@ impl Instance { // Guest tasks need to block if they have not yet returned or // cancelled, even as a result of the event delivery above. needs_block = !store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(guest_task)? .returned_or_cancelled() } else { @@ -3895,7 +4018,7 @@ impl Instance { // .. fall through to determine what event's in store for us. } - let event = waitable.take_event(store.concurrent_state_mut())?; + let event = waitable.take_event(store.concurrent_state_mut()?)?; if let Some(Event::Subtask { status: status @ (Status::Returned | Status::ReturnCancelled), }) = event @@ -5039,7 +5162,11 @@ impl From> for CurrentThread { /// Represents the Component Model Async state of a store. pub struct ConcurrentState { /// The currently running thread, if any. - current_thread: CurrentThread, + /// + /// Note that we lazily materialize threads on-demand and this field is not + /// necessarily up-to-date. The `StoreOpaque::current_thread` method should + /// be preferred over directly accessing this field. + unforced_current_thread: CurrentThread, /// The set of pending host and background tasks, if any. /// @@ -5101,7 +5228,7 @@ pub struct ConcurrentState { impl Default for ConcurrentState { fn default() -> Self { Self { - current_thread: CurrentThread::None, + unforced_current_thread: CurrentThread::None, table: AlwaysMut::new(ResourceTable::new()), futures: AlwaysMut::new(Some(FuturesUnordered::new())), high_priority: Vec::new(), @@ -5290,18 +5417,6 @@ impl ConcurrentState { } } - /// Returns whether there's a pending cancellation on the current guest thread, - /// consuming the event if so. - fn take_pending_cancellation(&mut self) -> Result { - let thread = self.current_guest_thread()?; - let task = self.get_mut(thread.task)?; - if let Some(Event::Cancelled) = task.event { - task.event.take(); - return Ok(true); - } - Ok(false) - } - fn check_blocking_for(&mut self, task: TableId) -> Result<()> { if self.may_block(task)? { Ok(()) @@ -5334,7 +5449,7 @@ impl ConcurrentState { /// Used by `ResourceTables` to record the scope of a borrow to get undone /// in the future. pub fn current_call_context_scope_id(&self) -> Result { - let (bits, is_host) = match self.current_thread { + let (bits, is_host) = match self.unforced_current_thread { CurrentThread::Guest(id) => (id.task.rep(), false), CurrentThread::Host(id) => (id.rep(), true), CurrentThread::None => bail_bug!("current thread is not set"), @@ -5343,20 +5458,6 @@ impl ConcurrentState { Ok((bits << 1) | u32::from(is_host)) } - fn current_guest_thread(&self) -> Result { - match self.current_thread.guest() { - Some(id) => Ok(*id), - None => bail_bug!("current thread is not a guest thread"), - } - } - - fn current_host_thread(&self) -> Result> { - match self.current_thread.host() { - Some(id) => Ok(id), - None => bail_bug!("current thread is not a host thread"), - } - } - fn futures_mut(&mut self) -> Result<&mut FuturesUnordered> { match self.futures.get_mut().as_mut() { Some(f) => Ok(f), @@ -5502,7 +5603,7 @@ impl TaskId { /// and can be resumed by other tasks for this component, so we mark the future as dropped /// and delete the task when all threads are done. pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> { - let task = store.concurrent_state_mut().get_mut(self.task)?; + let task = store.concurrent_state_mut()?.get_mut(self.task)?; let delete = if !task.already_lowered_parameters() { store.cancel_guest_subtask_without_lowered_parameters( self.runtime_instance, @@ -5514,7 +5615,7 @@ impl TaskId { task.ready_to_delete() }; if delete { - Waitable::Guest(self.task).delete_from(store.concurrent_state_mut())? + Waitable::Guest(self.task).delete_from(store.concurrent_state_mut()?)? } Ok(()) } @@ -5554,12 +5655,12 @@ pub(crate) fn prepare_call( .map(SendSyncPtr::new); let string_encoding = options.string_encoding; let token = StoreToken::new(store.as_context_mut()); - let state = store.0.concurrent_state_mut(); + let caller = store.0.current_thread()?; + let state = store.0.concurrent_state_mut()?; let (tx, rx) = oneshot::channel(); let instance = handle.instance().runtime_instance(component_instance); - let caller = state.current_thread; let thread = GuestTask::new( state, Box::new(for_any_lower(move |store, params| { diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs index ed5687b8b209..398f9e7f1e59 100644 --- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs +++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs @@ -417,7 +417,7 @@ impl<'a, T, B> Destination<'a, T, B> { } fn remaining_(&self, store: &mut StoreOpaque) -> Result> { - let transmit = store.concurrent_state_mut().get_mut(self.id)?; + let transmit = store.concurrent_state_mut()?.get_mut(self.id)?; if let &ReadState::GuestReady { count, .. } = &transmit.read { let &WriteState::HostReady { guest_offset, .. } = &transmit.write else { @@ -500,7 +500,7 @@ impl DirectDestination<'_, D> { .store .as_context_mut() .0 - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(self.id)?; let &ReadState::GuestReady { @@ -551,7 +551,7 @@ impl DirectDestination<'_, D> { .store .as_context_mut() .0 - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(self.id)?; let ReadState::GuestReady { @@ -890,7 +890,7 @@ impl<'a, T> Source<'a, T> { buffer.move_from(*input, count); } else { let store = store.as_context_mut(); - let transmit = store.0.concurrent_state_mut().get_mut(self.id)?; + let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?; let &ReadState::HostReady { guest_offset, .. } = &transmit.read else { bail_bug!("expected ReadState::HostReady"); @@ -919,7 +919,7 @@ impl<'a, T> Source<'a, T> { count.as_usize() - guest_offset.as_usize(), )?; - let transmit = store.0.concurrent_state_mut().get_mut(self.id)?; + let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?; let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else { bail_bug!("expected ReadState::HostReady"); @@ -947,7 +947,7 @@ impl<'a, T> Source<'a, T> { where T: 'static, { - let transmit = store.concurrent_state_mut().get_mut(self.id)?; + let transmit = store.concurrent_state_mut()?.get_mut(self.id)?; if let &WriteState::GuestReady { count, .. } = &transmit.write { let &ReadState::HostReady { guest_offset, .. } = &transmit.read else { @@ -1010,7 +1010,7 @@ impl DirectSource<'_, D> { .store .as_context_mut() .0 - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(self.id)?; let &WriteState::GuestReady { @@ -1061,7 +1061,7 @@ impl DirectSource<'_, D> { .store .as_context_mut() .0 - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(self.id)?; let WriteState::GuestReady { @@ -1721,7 +1721,7 @@ impl StreamReader { /// belong to the specified `store`. pub fn try_into(mut self, mut store: impl AsContextMut) -> Result { let store = store.as_context_mut(); - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut_already_forced_current_thread(); let id = state.get_mut(self.id).unwrap().state; if let WriteState::HostReady { try_into, .. } = &state.get_mut(id).unwrap().write { match try_into(TypeId::of::()) { @@ -2313,7 +2313,7 @@ impl StoreOpaque { let future = async move { let stream_state = future.await?; tls::get(|store| { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let transmit = state.get_mut(id)?; let ReadState::HostReady { consume, @@ -2343,7 +2343,8 @@ impl StoreOpaque { }) }; - self.concurrent_state_mut().push_future(future.boxed()); + self.concurrent_state_mut_already_forced_current_thread() + .push_future(future.boxed()); } fn pipe_to_guest( @@ -2355,7 +2356,7 @@ impl StoreOpaque { let future = async move { let stream_state = future.await?; tls::get(|store| { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let transmit = state.get_mut(id)?; let WriteState::HostReady { produce, @@ -2387,12 +2388,13 @@ impl StoreOpaque { }) }; - self.concurrent_state_mut().push_future(future.boxed()); + self.concurrent_state_mut_already_forced_current_thread() + .push_future(future.boxed()); } /// Drop the read end of a stream or future read from the host. fn host_drop_reader(&mut self, id: TableId, kind: TransmitKind) -> Result<()> { - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; Waitable::Transmit(id).join(state, None)?; let transmit_id = state.get_mut(id)?.state; let transmit = state @@ -2469,7 +2471,7 @@ impl StoreOpaque { id: TableId, on_drop_open: Option Result<()>>, ) -> Result<()> { - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; Waitable::Transmit(id).join(state, None)?; let transmit_id = state.get_mut(id)?.state; let transmit = state @@ -2500,7 +2502,7 @@ impl StoreOpaque { WriteState::Dropped => bail_bug!("write state is already dropped"), } - let transmit = self.concurrent_state_mut().get_mut(transmit_id)?; + let transmit = self.concurrent_state_mut()?.get_mut(transmit_id)?; // If the existing read state is dropped, then there's nothing to read // and we can keep it that way. @@ -2522,7 +2524,7 @@ impl StoreOpaque { // represent that a read must be performed ReadState::GuestReady { ty, handle, .. } => { // Ensure the final read of the guest is queued, with appropriate closure indicator - self.concurrent_state_mut().update_event( + self.concurrent_state_mut()?.update_event( read_handle.rep(), match ty { TransmitIndex::Future(ty) => Event::FutureRead { @@ -2539,7 +2541,7 @@ impl StoreOpaque { // If the read state is open, then there are no registered readers of the stream/future ReadState::Open => { - self.concurrent_state_mut().update_event( + self.concurrent_state_mut()?.update_event( read_handle.rep(), match on_drop_open { Some(_) => Event::FutureRead { @@ -2562,7 +2564,7 @@ impl StoreOpaque { // this event. ReadState::Dropped | ReadState::HostReady { .. } | ReadState::HostToHost { .. } => { log::trace!("host_drop_writer delete {transmit_id:?}"); - self.concurrent_state_mut().delete_transmit(transmit_id)?; + self.concurrent_state_mut()?.delete_transmit(transmit_id)?; } } Ok(()) @@ -2572,7 +2574,7 @@ impl StoreOpaque { &mut self, id: TableId, ) -> Result { - let state = self.concurrent_state_mut(); + let state = self.concurrent_state_mut()?; let state_id = state.get_mut(id)?.state; Ok(state.get_mut(state_id)?.origin) } @@ -2588,7 +2590,7 @@ impl StoreContextMut<'_, T> { P::Item: func::Lower, { let token = StoreToken::new(self.as_context_mut()); - let state = self.0.concurrent_state_mut(); + let state = self.0.concurrent_state_mut()?; let (_, read) = state.new_transmit(TransmitOrigin::Host)?; let producer = Arc::new(LockedState::new((Box::pin(producer), P::Buffer::default()))); let id = state.get_mut(read)?.state; @@ -2604,7 +2606,7 @@ impl StoreContextMut<'_, T> { let (result, cancelled) = if buffer.remaining().is_empty() { future::poll_fn(|cx| { tls::get(|store| { - let transmit = store.concurrent_state_mut().get_mut(id)?; + let transmit = store.concurrent_state_mut()?.get_mut(id)?; let &WriteState::HostReady { cancel, .. } = &transmit.write else { bail_bug!("expected WriteState::HostReady") @@ -2635,7 +2637,7 @@ impl StoreContextMut<'_, T> { cancel, ); - let transmit = store.concurrent_state_mut().get_mut(id)?; + let transmit = store.concurrent_state_mut()?.get_mut(id)?; let host_offset = if let ( Some(host_buffer), @@ -2686,7 +2688,7 @@ impl StoreContextMut<'_, T> { }; let (guest_offset, host_offset, count) = tls::get(|store| { - let transmit = store.concurrent_state_mut().get_mut(id)?; + let transmit = store.concurrent_state_mut()?.get_mut(id)?; let (count, host_offset) = match &transmit.read { &ReadState::GuestReady { count, .. } => (count.as_u32(), 0), &ReadState::HostToHost { limit, .. } => (1, limit), @@ -2773,7 +2775,7 @@ impl StoreContextMut<'_, T> { consumer: C, ) -> Result<()> { let token = StoreToken::new(self.as_context_mut()); - let state = self.0.concurrent_state_mut(); + let state = self.0.concurrent_state_mut()?; let id = state.get_mut(id)?.state; let transmit = state.get_mut(id)?; let consumer = Arc::new(LockedState::new(Box::pin(consumer))); @@ -2787,7 +2789,7 @@ impl StoreContextMut<'_, T> { let (result, cancelled) = future::poll_fn(|cx| { tls::get(|store| { - let cancel = match &store.concurrent_state_mut().get_mut(id)?.read { + let cancel = match &store.concurrent_state_mut()?.get_mut(id)?.read { &ReadState::HostReady { cancel, .. } => cancel, ReadState::Open => false, _ => bail_bug!("unexpected read state"), @@ -2807,7 +2809,7 @@ impl StoreContextMut<'_, T> { cancel_waker, cancel, .. - } = &mut store.concurrent_state_mut().get_mut(id)?.read + } = &mut store.concurrent_state_mut()?.get_mut(id)?.read { if poll.is_pending() { *cancel_waker = Some(cx.waker().clone()); @@ -2823,7 +2825,7 @@ impl StoreContextMut<'_, T> { .await?; let (guest_offset, count) = tls::get(|store| { - let transmit = store.concurrent_state_mut().get_mut(id)?; + let transmit = store.concurrent_state_mut()?.get_mut(id)?; Ok(( match &transmit.read { &ReadState::HostReady { guest_offset, .. } => guest_offset, @@ -2858,7 +2860,7 @@ impl StoreContextMut<'_, T> { if let TransmitKind::Future = kind { tls::get(|store| { - store.concurrent_state_mut().get_mut(id)?.done = true; + store.concurrent_state_mut()?.get_mut(id)?.done = true; crate::error::Ok(()) })?; } @@ -2933,7 +2935,7 @@ impl StoreContextMut<'_, T> { loop { if tls::get(|store| { crate::error::Ok(matches!( - store.concurrent_state_mut().get_mut(id)?.read, + store.concurrent_state_mut()?.get_mut(id)?.read, ReadState::Dropped )) })? { @@ -2951,7 +2953,7 @@ impl StoreContextMut<'_, T> { } } .map(move |result| { - tls::get(|store| store.concurrent_state_mut().delete_transmit(id))?; + tls::get(|store| store.concurrent_state_mut()?.delete_transmit(id))?; result }); @@ -2973,7 +2975,7 @@ async fn write Result<()> { let (read, guest_offset) = tls::get(|store| { - let transmit = store.concurrent_state_mut().get_mut(id)?; + let transmit = store.concurrent_state_mut()?.get_mut(id)?; let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write { Some(guest_offset) @@ -3006,7 +3008,7 @@ async fn write r, Err(oneshot::Canceled) => bail_bug!("work cancelled"), @@ -3063,7 +3066,7 @@ async fn write ReadState::Dropped, StreamResult::Completed | StreamResult::Cancelled => ReadState::HostToHost { accept, @@ -3144,7 +3147,7 @@ impl Instance { cancel: bool, ) -> Result { let mut future = consume(); - store.concurrent_state_mut().get_mut(transmit_id)?.read = ReadState::HostReady { + store.concurrent_state_mut()?.get_mut(transmit_id)?.read = ReadState::HostReady { consume, guest_offset, cancel, @@ -3158,7 +3161,7 @@ impl Instance { Ok(match poll { Poll::Ready(state) => { - let transmit = store.concurrent_state_mut().get_mut(transmit_id)?; + let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?; let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else { bail_bug!("expected ReadState::HostReady") }; @@ -3186,7 +3189,7 @@ impl Instance { cancel: bool, ) -> Result { let mut future = produce(); - store.concurrent_state_mut().get_mut(transmit_id)?.write = WriteState::HostReady { + store.concurrent_state_mut()?.get_mut(transmit_id)?.write = WriteState::HostReady { produce, try_into, guest_offset, @@ -3201,7 +3204,7 @@ impl Instance { Ok(match poll { Poll::Ready(state) => { - let transmit = store.concurrent_state_mut().get_mut(transmit_id)?; + let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?; let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else { bail_bug!("expected WriteState::HostReady") }; @@ -3506,7 +3509,7 @@ impl Instance { *state = TransmitLocalState::Busy; let transmit_handle = TableId::::new(rep); - let concurrent_state = store.0.concurrent_state_mut(); + let concurrent_state = store.0.concurrent_state_mut()?; let transmit_id = concurrent_state.get_mut(transmit_handle)?.state; let transmit = concurrent_state.get_mut(transmit_id)?; log::trace!( @@ -3615,7 +3618,7 @@ impl Instance { Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?, None => 0, }; - let concurrent_state = store.0.concurrent_state_mut(); + let concurrent_state = store.0.concurrent_state_mut()?; if read_complete { let total = if let Some(Event::StreamRead { code: ReturnCode::Completed(old_total), @@ -3760,8 +3763,8 @@ impl Instance { *state = TransmitLocalState::Busy; let transmit_handle = TableId::::new(rep); - let concurrent_state = store.0.concurrent_state_mut(); - let caller_thread = concurrent_state.current_guest_thread()?; + let caller_thread = store.0.current_guest_thread()?; + let concurrent_state = store.0.concurrent_state_mut()?; let transmit_id = concurrent_state.get_mut(transmit_handle)?.state; let transmit = concurrent_state.get_mut(transmit_id)?; log::trace!( @@ -3853,7 +3856,7 @@ impl Instance { Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?, None => 0, }; - let concurrent_state = store.0.concurrent_state_mut(); + let concurrent_state = store.0.concurrent_state_mut()?; if write_complete { let total = if let Some(Event::StreamWrite { @@ -3928,7 +3931,7 @@ impl Instance { )?; if let (TransmitIndex::Future(_), ReturnCode::Completed(_)) = (ty, code) { - store.0.concurrent_state_mut().get_mut(transmit_id)?.done = true; + store.0.concurrent_state_mut()?.get_mut(transmit_id)?.done = true; } code @@ -3970,7 +3973,7 @@ impl Instance { ) -> Result { let waitable = Waitable::Transmit(handle); store.wait_for_event(waitable)?; - let event = waitable.take_event(store.concurrent_state_mut())?; + let event = waitable.take_event(store.concurrent_state_mut()?)?; if let Some(event @ (Event::StreamWrite { code, .. } | Event::FutureWrite { code, .. })) = event { @@ -3988,7 +3991,7 @@ impl Instance { transmit_id: TableId, async_: bool, ) -> Result { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let transmit = state.get_mut(transmit_id)?; log::trace!( "host_cancel_write state {transmit_id:?}; write state {:?} read state {:?}", @@ -4024,7 +4027,7 @@ impl Instance { ReturnCode::Blocked } else { let handle = store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(transmit_id)? .write_handle; self.wait_for_write(store, handle)? @@ -4034,7 +4037,7 @@ impl Instance { }; if !matches!(code, ReturnCode::Blocked) { - let transmit = store.concurrent_state_mut().get_mut(transmit_id)?; + let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?; match &transmit.write { WriteState::GuestReady { .. } => { @@ -4057,7 +4060,7 @@ impl Instance { ) -> Result { let waitable = Waitable::Transmit(handle); store.wait_for_event(waitable)?; - let event = waitable.take_event(store.concurrent_state_mut())?; + let event = waitable.take_event(store.concurrent_state_mut()?)?; if let Some(event @ (Event::StreamRead { code, .. } | Event::FutureRead { code, .. })) = event { @@ -4075,7 +4078,7 @@ impl Instance { transmit_id: TableId, async_: bool, ) -> Result { - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let transmit = state.get_mut(transmit_id)?; log::trace!( "host_cancel_read state {transmit_id:?}; read state {:?} write state {:?}", @@ -4111,7 +4114,7 @@ impl Instance { ReturnCode::Blocked } else { let handle = store - .concurrent_state_mut() + .concurrent_state_mut()? .get_mut(transmit_id)? .read_handle; self.wait_for_read(store, handle)? @@ -4121,7 +4124,7 @@ impl Instance { }; if !matches!(code, ReturnCode::Blocked) { - let transmit = store.concurrent_state_mut().get_mut(transmit_id)?; + let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?; match &transmit.read { ReadState::GuestReady { .. } => { @@ -4167,7 +4170,7 @@ impl Instance { } TransmitLocalState::Busy => {} } - let transmit_id = store.concurrent_state_mut().get_mut(id)?.state; + let transmit_id = store.concurrent_state_mut()?.get_mut(id)?.state; let code = self.cancel_write(store, transmit_id, async_)?; if !matches!(code, ReturnCode::Blocked) { let state = @@ -4208,7 +4211,7 @@ impl Instance { } TransmitLocalState::Busy => {} } - let transmit_id = store.concurrent_state_mut().get_mut(id)?.state; + let transmit_id = store.concurrent_state_mut()?.get_mut(id)?.state; let code = self.cancel_read(store, transmit_id, async_)?; if !matches!(code, ReturnCode::Blocked) { let state = @@ -4260,7 +4263,7 @@ impl Instance { // Create a new ErrorContext that is tracked along with other concurrent state let err_ctx = ErrorContextState { debug_msg }; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let table_id = state.push(err_ctx)?; let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(table_id.rep()); @@ -4301,7 +4304,7 @@ impl Instance { .table_for_error_context(ty) .error_context_rep(err_ctx_handle)?; - let state = store.0.concurrent_state_mut(); + let state = store.0.concurrent_state_mut()?; // Get the state associated with the error context let ErrorContextState { debug_msg } = state.get_mut(TableId::::new(handle_table_id_rep))?; @@ -4400,7 +4403,7 @@ impl Instance { /// `TransmitIndex` belongs. fn guest_new(self, store: &mut StoreOpaque, ty: TransmitIndex) -> Result { let (write, read) = store - .concurrent_state_mut() + .concurrent_state_mut()? .new_transmit(TransmitOrigin::guest(self.id().instance(), ty))?; let table = self.id().get_mut(store).table_for_transmit(ty); @@ -4415,7 +4418,7 @@ impl Instance { ), }; - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; state.get_mut(read)?.common.handle = Some(read_handle); state.get_mut(write)?.common.handle = Some(write_handle); @@ -4440,7 +4443,7 @@ impl Instance { let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(rep); - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let Some(GlobalErrorContextRefCount(global_ref_count)) = state .global_error_context_ref_counts .get_mut(&global_ref_count_idx) @@ -4572,7 +4575,7 @@ impl Instance { // as the new component has essentially created a new reference that will // be dropped/handled independently let global_ref_count = store - .concurrent_state_mut() + .concurrent_state_mut()? .global_error_context_ref_counts .get_mut(&TypeComponentGlobalErrorContextTableIndex::from_u32(rep)) .context("global ref count present for existing (sub)component error context")?; @@ -4884,7 +4887,7 @@ impl Waitable { }; let transmit_handle = TableId::::new(rep); - let state = store.concurrent_state_mut(); + let state = store.concurrent_state_mut()?; let transmit_id = state.get_mut(transmit_handle)?.state; let transmit = state.get_mut(transmit_id)?; diff --git a/crates/wasmtime/src/runtime/component/func.rs b/crates/wasmtime/src/runtime/component/func.rs index 423bb60ebb04..a16fc33b6b03 100644 --- a/crates/wasmtime/src/runtime/component/func.rs +++ b/crates/wasmtime/src/runtime/component/func.rs @@ -499,7 +499,7 @@ impl Func { // as they're required to have been dropped by this point. store .0 - .component_resource_tables(Some(self.instance)) + .component_resource_tables(Some(self.instance))? .validate_scope_exit()?; // SAFETY: We're relying on the correctness of the structure of diff --git a/crates/wasmtime/src/runtime/component/func/options.rs b/crates/wasmtime/src/runtime/component/func/options.rs index d02fc7e4db31..51aab7e0d6f9 100644 --- a/crates/wasmtime/src/runtime/component/func/options.rs +++ b/crates/wasmtime/src/runtime/component/func/options.rs @@ -221,7 +221,7 @@ impl<'a, T: 'static> LowerContext<'a, T> { ty: TypeResourceTableIndex, rep: u32, ) -> Result { - self.resource_tables().guest_resource_lower_own(rep, ty) + self.resource_tables()?.guest_resource_lower_own(rep, ty) } /// Lowers a `borrow` resource into the guest, converting the `rep` to a @@ -242,19 +242,19 @@ impl<'a, T: 'static> LowerContext<'a, T> { if self.instance().resource_owned_by_own_instance(ty) { return Ok(rep); } - self.resource_tables().guest_resource_lower_borrow(rep, ty) + self.resource_tables()?.guest_resource_lower_borrow(rep, ty) } /// Lifts a host-owned `own` resource at the `idx` specified into the /// representation of that resource. pub fn host_resource_lift_own(&mut self, idx: HostResourceIndex) -> Result { - self.resource_tables().host_resource_lift_own(idx) + self.resource_tables()?.host_resource_lift_own(idx) } /// Lifts a host-owned `borrow` resource at the `idx` specified into the /// representation of that resource. pub fn host_resource_lift_borrow(&mut self, idx: HostResourceIndex) -> Result { - self.resource_tables().host_resource_lift_borrow(idx) + self.resource_tables()?.host_resource_lift_borrow(idx) } /// Lowers a resource into the host-owned table, returning the index it was @@ -268,7 +268,7 @@ impl<'a, T: 'static> LowerContext<'a, T> { dtor: Option>, instance: Option, ) -> Result { - self.resource_tables() + self.resource_tables()? .host_resource_lower_own(rep, dtor, instance) } @@ -283,18 +283,18 @@ impl<'a, T: 'static> LowerContext<'a, T> { InstanceType::new(self.instance()) } - fn resource_tables(&mut self) -> HostResourceTables<'_> { + fn resource_tables(&mut self) -> Result> { let (tables, data) = self .store .0 - .component_resource_tables_and_host_resource_data(Some(self.instance)); - HostResourceTables::from_parts(tables, data) + .component_resource_tables_and_host_resource_data(Some(self.instance))?; + Ok(HostResourceTables::from_parts(tables, data)) } /// See [`HostResourceTables::validate_scope_exit`]. #[inline] pub fn validate_scope_exit(&mut self) -> Result<()> { - self.resource_tables().validate_scope_exit() + self.resource_tables()?.validate_scope_exit() } } diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs index 262d181f9819..13afe2c77014 100644 --- a/crates/wasmtime/src/runtime/component/instance.rs +++ b/crates/wasmtime/src/runtime/component/instance.rs @@ -381,7 +381,7 @@ impl Instance { rep: u32, ) -> Result { store - .component_resource_tables(Some(self)) + .component_resource_tables(Some(self))? .resource_new(TypedResource::Component { ty, rep }) } @@ -394,7 +394,7 @@ impl Instance { index: u32, ) -> Result { store - .component_resource_tables(Some(self)) + .component_resource_tables(Some(self))? .resource_rep(TypedResourceIndex::Component { ty, index }) } @@ -406,7 +406,7 @@ impl Instance { index: u32, ) -> Result> { store - .component_resource_tables(Some(self)) + .component_resource_tables(Some(self))? .resource_drop(TypedResourceIndex::Component { ty, index }) } @@ -417,7 +417,7 @@ impl Instance { src: TypeResourceTableIndex, dst: TypeResourceTableIndex, ) -> Result { - let mut tables = store.component_resource_tables(Some(self)); + let mut tables = store.component_resource_tables(Some(self))?; let rep = tables.resource_lift_own(TypedResourceIndex::Component { ty: src, index })?; tables.resource_lower_own(TypedResource::Component { ty: dst, rep }) } @@ -430,7 +430,7 @@ impl Instance { dst: TypeResourceTableIndex, ) -> Result { let dst_owns_resource = self.id().get(store).resource_owned_by_own_instance(dst); - let mut tables = store.component_resource_tables(Some(self)); + let mut tables = store.component_resource_tables(Some(self))?; let rep = tables.resource_lift_borrow(TypedResourceIndex::Component { ty: src, index })?; // Implement `lower_borrow`'s special case here where if a borrow's // resource type is owned by `dst` then the destination receives the diff --git a/crates/wasmtime/src/runtime/component/resources/any.rs b/crates/wasmtime/src/runtime/component/resources/any.rs index cc84cecb895a..ecb69e4822e8 100644 --- a/crates/wasmtime/src/runtime/component/resources/any.rs +++ b/crates/wasmtime/src/runtime/component/resources/any.rs @@ -109,7 +109,7 @@ impl ResourceAny { D: PartialEq + Send + Sync + Copy + 'static, { let store = store.as_context_mut(); - let mut tables = HostResourceTables::new_host(store.0); + let mut tables = HostResourceTables::new_host(store.0)?; let ResourceAny { idx, ty, owned } = self; let ty = T::typecheck(ty).ok_or_else(|| crate::format_err!("resource type mismatch"))?; if owned { @@ -189,7 +189,7 @@ impl ResourceAny { // // This could fail if the index is invalid or if this is removing an // `Own` entry which is currently being borrowed. - let pair = HostResourceTables::new_host(store.0).host_resource_drop(self.idx)?; + let pair = HostResourceTables::new_host(store.0)?.host_resource_drop(self.idx)?; let (rep, slot) = match (pair, self.owned) { (Some(pair), true) => pair, diff --git a/crates/wasmtime/src/runtime/component/resources/host.rs b/crates/wasmtime/src/runtime/component/resources/host.rs index e1d363c696b4..70f43f06f7bc 100644 --- a/crates/wasmtime/src/runtime/component/resources/host.rs +++ b/crates/wasmtime/src/runtime/component/resources/host.rs @@ -292,7 +292,7 @@ where } = self; let store = store.as_context_mut(); - let mut tables = HostResourceTables::new_host(store.0); + let mut tables = HostResourceTables::new_host(store.0)?; let (idx, owned) = match state.get() { ResourceState::Borrow => (tables.host_resource_lower_borrow(rep)?, false), ResourceState::NotInTable => { diff --git a/crates/wasmtime/src/runtime/component/resources/host_tables.rs b/crates/wasmtime/src/runtime/component/resources/host_tables.rs index a8ba4809cb8d..4835e538e0b6 100644 --- a/crates/wasmtime/src/runtime/component/resources/host_tables.rs +++ b/crates/wasmtime/src/runtime/component/resources/host_tables.rs @@ -96,9 +96,9 @@ impl HostResourceIndex { } impl<'a> HostResourceTables<'a> { - pub fn new_host(store: &'a mut StoreOpaque) -> HostResourceTables<'a> { - let (tables, data) = store.component_resource_tables_and_host_resource_data(None); - HostResourceTables::from_parts(tables, data) + pub fn new_host(store: &'a mut StoreOpaque) -> Result> { + let (tables, data) = store.component_resource_tables_and_host_resource_data(None)?; + Ok(HostResourceTables::from_parts(tables, data)) } pub fn from_parts( diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rs index 46d7fe0bfa48..dc9457424649 100644 --- a/crates/wasmtime/src/runtime/component/store.rs +++ b/crates/wasmtime/src/runtime/component/store.rs @@ -136,7 +136,7 @@ impl ComponentStoreData { let mut fibers = Vec::new(); let mut futures = Vec::new(); store - .concurrent_state_mut() + .concurrent_state_mut_without_forcing_current_thread() .take_fibers_and_futures(&mut fibers, &mut futures); for mut fiber in fibers { @@ -292,8 +292,11 @@ impl StoreOpaque { &mut self.store_data_mut().components } - pub(crate) fn component_task_state_mut(&mut self) -> &mut ComponentTaskState { - &mut self.component_data_mut().task_state + pub(crate) fn component_task_state_mut(&mut self) -> Result<&mut ComponentTaskState> { + #[cfg(feature = "component-model-async")] + let _ = self.current_thread()?; + + Ok(&mut self.component_data_mut().task_state) } pub(crate) fn push_component_instance(&mut self, instance: Instance) { @@ -318,11 +321,29 @@ impl StoreOpaque { } #[cfg(feature = "component-model-async")] - pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState { + pub(crate) fn concurrent_state_mut_without_forcing_current_thread( + &mut self, + ) -> &mut ConcurrentState { debug_assert!(self.concurrency_support()); self.component_data_mut().task_state.concurrent_state_mut() } + #[cfg(feature = "component-model-async")] + pub(crate) fn concurrent_state_mut_already_forced_current_thread( + &mut self, + ) -> &mut ConcurrentState { + debug_assert!(self.concurrency_support()); + debug_assert!(!self.vm_store_context().current_thread.is_deferred()); + self.concurrent_state_mut_without_forcing_current_thread() + } + + #[cfg(feature = "component-model-async")] + pub(crate) fn concurrent_state_mut(&mut self) -> Result<&mut ConcurrentState> { + debug_assert!(self.concurrency_support()); + self.current_thread()?; + Ok(self.component_data_mut().task_state.concurrent_state_mut()) + } + #[inline] #[cfg(feature = "component-model-async")] pub(crate) fn concurrency_support(&self) -> bool { @@ -357,18 +378,22 @@ impl StoreOpaque { pub(crate) fn component_resource_tables( &mut self, instance: Option, - ) -> vm::component::ResourceTables<'_> { - self.component_resource_tables_and_host_resource_data(instance) - .0 + ) -> Result> { + Ok(self + .component_resource_tables_and_host_resource_data(instance)? + .0) } pub(crate) fn component_resource_tables_and_host_resource_data( &mut self, instance: Option, - ) -> ( + ) -> Result<( vm::component::ResourceTables<'_>, &mut crate::component::HostResourceData, - ) { + )> { + #[cfg(feature = "component-model-async")] + let _ = self.current_thread()?; + let store_id = self.id(); let data = self.component_data_mut(); let guest = instance.map(|i| { @@ -381,14 +406,14 @@ impl StoreOpaque { .instance_states() }); - ( + Ok(( vm::component::ResourceTables { host_table: &mut data.component_host_table, task_state: &mut data.task_state, guest, }, &mut data.host_resource_data, - ) + )) } pub(crate) fn enter_call_not_concurrent(&mut self) -> Result<()> { @@ -419,7 +444,10 @@ impl StoreOpaque { #[cfg(feature = "component-model-async")] fn concurrent_resource_table(&mut self) -> Option<&mut ResourceTable> { if self.concurrency_support() { - Some(self.concurrent_state_mut().table()) + Some( + self.concurrent_state_mut_without_forcing_current_thread() + .table(), + ) } else { None } diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 22914fbdda5f..46c06f7b15b8 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -2,8 +2,8 @@ use crate::error::OutOfMemory; use crate::prelude::*; use crate::runtime::vm::{ self, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, - VMCommonStackInformation, VMContext, VMFuncRef, VMFunctionImport, VMOpaqueContext, - VMStoreContext, + VMCommonStackInformation, VMContext, VMFuncRef, VMFunctionImport, VMLazyThread, + VMOpaqueContext, VMStoreContext, }; use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreId, StoreOpaque}; use crate::type_registry::RegisteredType; @@ -1476,6 +1476,16 @@ pub(crate) fn invoke_wasm_and_catch_traps( #[cfg(feature = "component-model")] if result.is_err() { store.0.set_trapped(); + + // A trap unwinds over any fused adapter frames without running their + // `exit-sync-call`s. That can leave `VMStoreContext::current_thread` + // pointing at an old, invalid `VMDeferredThread` inside an old, + // since-unwound stack frame. Therefore, we must reset `current_thread` + // to avoid potential use-after-free bugs. + let vm_store_context = store.0.vm_store_context_mut(); + if vm_store_context.current_thread.is_deferred() { + vm_store_context.current_thread = VMLazyThread::forced(); + } } core::mem::drop(previous_runtime_state); store.0.call_hook(CallHook::ReturningFromWasm)?; diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index bda01cd5c2e7..ac690ffd8ec8 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -2472,7 +2472,9 @@ unsafe impl VMStore for StoreInner { } #[cfg(feature = "component-model")] - fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState { + fn component_task_state_mut( + &mut self, + ) -> Result<&mut crate::component::store::ComponentTaskState> { StoreOpaque::component_task_state_mut(self) } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index e97b4259ad24..314c1aa99c1c 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -128,8 +128,8 @@ pub use crate::runtime::vm::traphandlers::*; pub use crate::runtime::vm::vmcontext::VMArrayCallFunction; pub use crate::runtime::vm::vmcontext::{ VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition, - VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, - VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw, + VMGlobalImport, VMGlobalKind, VMLazyThread, VMMemoryDefinition, VMMemoryImport, + VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw, }; #[cfg(has_custom_sync)] pub(crate) use sys::capi; @@ -219,7 +219,9 @@ pub unsafe trait VMStore: 'static { /// Metadata required for resources for the component model. #[cfg(feature = "component-model")] - fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState; + fn component_task_state_mut( + &mut self, + ) -> Result<&mut crate::component::store::ComponentTaskState>; #[cfg(feature = "component-model-async")] fn component_async_store( diff --git a/crates/wasmtime/src/runtime/vm/component/libcalls.rs b/crates/wasmtime/src/runtime/vm/component/libcalls.rs index 094b5562c2d6..2484a09db19b 100644 --- a/crates/wasmtime/src/runtime/vm/component/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/component/libcalls.rs @@ -688,7 +688,7 @@ fn enter_sync_call( fn exit_sync_call(store: &mut dyn VMStore, instance: Instance) -> Result<()> { store - .component_resource_tables(Some(instance)) + .component_resource_tables(Some(instance))? .validate_scope_exit()?; store.exit_guest_sync_call() } diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 83844fb4eeff..281e42382cc0 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -1227,6 +1227,14 @@ pub struct VMStoreContext { /// /// This is saved/restored when threads are swapped in the component model. pub component_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS], + + /// JIT-visible current thread for the component model's sync-to-sync + /// adapter fast path. + /// + /// Like `component_context`, this is unconditionally present to keep + /// `VMOffsets` logic unconditional even though it is only used when + /// `component-model-async` is enabled. + pub current_thread: VMLazyThread, } impl VMStoreContext { @@ -1312,6 +1320,7 @@ impl Default for VMStoreContext { async_guard_range: ptr::null_mut()..ptr::null_mut(), store_data: VmPtr::dangling(), component_context: [0; NUM_COMPONENT_CONTEXT_SLOTS], + current_thread: VMLazyThread::none(), } } } @@ -1386,6 +1395,10 @@ mod test_vmstore_context { offset_of!(VMStoreContext, component_context), usize::from(offsets.ptr.vmstore_context_component_context_slot(0)) ); + assert_eq!( + offset_of!(VMStoreContext, current_thread), + usize::from(offsets.ptr.vmstore_context_current_thread()) + ); // Make sure that the calculation for the size of a slot is also // accurate. @@ -1399,6 +1412,159 @@ mod test_vmstore_context { } } +/// JIT-visible representation of the store's current thread for the component +/// model, encoded as a single pointer-sized integer so that generated JIT code +/// can load, store, and compare it with a handful of instructions. +/// +/// This is the inline fast-path counterpart to the host-side `CurrentThread`: a +/// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to +/// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a +/// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred +/// thread into a real one only when it actually needs it; see +/// `StoreOpaque::force_current_thread`. +/// +/// This type is a bitpacked equivalent of the following logical `enum`: +/// +/// ```ignore +/// enum VMLazyThread { +/// /// No thread. +/// None, +/// +/// /// The lazy thread was promoted and materialized; get it from +/// /// `ConcurrentState::current_thread`. +/// Forced, +/// +/// /// The lazy thread has not been materialized, here is a pointer to the +/// /// stack-allocated data needed to do force that promotion. +/// Deferred(*mut VMDeferredThread), +/// } +/// ``` +// +// Bitpacking details: +// +// * `None`: `0` +// +// * `Forced`: A non-zero value with its low-bit set. +// +// * `Deferred`: A non-zero value with its low-bit clear. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct VMLazyThread(usize); + +impl VMLazyThread { + const NONE: usize = 0; + const FORCED: usize = 1; + + /// There is no current thread. + pub const fn none() -> Self { + Self(Self::NONE) + } + + /// A lazy thread that has already been promoted. + pub const fn forced() -> Self { + Self(Self::FORCED) + } + + /// A deferred thread referencing the given on-stack [`VMDeferredThread`]. + pub fn deferred(ptr: *mut VMDeferredThread) -> Self { + let bits = ptr as usize; + debug_assert_ne!(bits, Self::NONE); + debug_assert_eq!(bits & Self::FORCED, 0); + Self(bits) + } + + /// Returns `true` if there is no current thread. + pub fn is_none(self) -> bool { + self.0 == Self::NONE + } + + /// Returns `true` if a deferred thread has been forced/promoted. + pub fn is_forced(self) -> bool { + self.0 & Self::FORCED != 0 + } + + /// Returns `true` if this is a deferred thread (i.e. neither `None` nor + /// forced). + pub fn is_deferred(self) -> bool { + self.0 != Self::NONE && self.0 & Self::FORCED == 0 + } + + /// Returns the deferred [`VMDeferredThread`] pointer if this is a deferred + /// thread. + pub fn as_deferred(self) -> Option<*mut VMDeferredThread> { + if self.is_deferred() { + Some(self.0 as *mut VMDeferredThread) + } else { + None + } + } +} + +/// A deferred component-model thread. +/// +/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast +/// path to defer the work that the `enter_sync_call` libcall would otherwise do +/// eagerly. +/// +/// The adapter allocates one of these in its own stack frame, links the +/// previous current-thread value to it via `parent`, and finally points +/// `VMStoreContext::current_thread` at it. When host code actually needs the +/// real thread, it walks the `parent` chain to materialize thread state (see +/// `StoreOpaque::force_current_thread`). +#[derive(Debug)] +#[repr(C)] +pub struct VMDeferredThread { + /// The previous value of `VMStoreContext::current_thread`. + pub parent: VMLazyThread, + /// The caller component instance (a deferred `enter_sync_call` argument). + pub caller_instance: u32, + /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg). + pub callee_async: u32, + /// The callee component instance (a deferred `enter_sync_call` argument). + pub callee_instance: u32, + /// The caller thread's `context.{get,set}` slots, saved on entry and + /// restored on the fast-path exit (or recovered while forcing). + pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS], +} + +#[cfg(test)] +mod test_vmdeferred_thread { + use super::*; + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets}; + + #[test] + fn deferred_thread_field_offsets() { + let module = Module::new(StaticModuleIndex::from_u32(0)); + let offsets = VMOffsets::new(HostPtr, &module); + let ptr = offsets.ptr; + assert_eq!( + offset_of!(VMDeferredThread, parent), + usize::from(ptr.vmdeferred_thread_parent()) + ); + assert_eq!( + offset_of!(VMDeferredThread, caller_instance), + usize::from(ptr.vmdeferred_thread_caller_instance()) + ); + assert_eq!( + offset_of!(VMDeferredThread, callee_async), + usize::from(ptr.vmdeferred_thread_callee_async()) + ); + assert_eq!( + offset_of!(VMDeferredThread, callee_instance), + usize::from(ptr.vmdeferred_thread_callee_instance()) + ); + assert_eq!( + offset_of!(VMDeferredThread, saved_context), + usize::from(ptr.vmdeferred_thread_saved_context(0)) + ); + assert_eq!( + size_of::(), + usize::from(ptr.size_of_vmdeferred_thread()) + ); + } +} + /// The VM "context", which is pointed to by the `vmctx` arg in Cranelift. /// This has information about globals, memories, tables, and other runtime /// state associated with the current instance. diff --git a/tests/all/component_model.rs b/tests/all/component_model.rs index b7e46fcc1377..e40780680528 100644 --- a/tests/all/component_model.rs +++ b/tests/all/component_model.rs @@ -23,6 +23,7 @@ mod nested; mod post_return; mod resources; mod strings; +mod sync_call_inline; #[derive(Copy, Clone)] enum ApiStyle { diff --git a/tests/all/component_model/async.rs b/tests/all/component_model/async.rs index 7df03ca8df8a..6b93b7b2e13a 100644 --- a/tests/all/component_model/async.rs +++ b/tests/all/component_model/async.rs @@ -1001,7 +1001,7 @@ async fn async_call_stack() -> Result<()> { linker.root().func_wrap( "a", |mut store: StoreContextMut>, (): ()| { - let stack = store.async_call_stack().collect::>(); + let stack = store.async_call_stack()?.collect::>(); assert_eq!(stack, [store.data().unwrap()]); Ok(()) }, @@ -1046,7 +1046,7 @@ async fn async_call_stack() -> Result<()> { linker.root().func_wrap( "a", |mut store: StoreContextMut>, (): ()| { - let stack = store.async_call_stack().collect::>(); + let stack = store.async_call_stack()?.collect::>(); assert_eq!(stack.len(), 2); assert_eq!(stack.last(), store.data().as_ref()); Ok(()) diff --git a/tests/all/component_model/sync_call_inline.rs b/tests/all/component_model/sync_call_inline.rs new file mode 100644 index 000000000000..62b42f080f42 --- /dev/null +++ b/tests/all/component_model/sync_call_inline.rs @@ -0,0 +1,385 @@ +//! Runtime tests for the guest-to-guest sync-call fast path (issue #12311). +//! +//! When concurrency support is enabled a fused sync-to-sync adapter lowers its +//! `enter-sync-call`/`exit-sync-call` intrinsics inline: `enter` pushes an +//! on-stack `VMDeferredThread` (saving and zeroing the caller's component +//! `context.{get,set}` slots), and `exit` pops it inline -- unless host code has +//! read the current thread in the meantime, in which case the deferred thread +//! is "forced" into a real one and `exit` falls back to the out-of-line libcall. +//! +//! Calling an imported *host* function from inside the callee is the canonical +//! way to force the deferred thread: every guest->host boundary runs +//! `StoreOpaque::force_current_thread`. These tests drive that path through the +//! public `TypedFunc` API and use `context.{get,set}` as a guest-observable +//! witness that the save / zero / restore / replay logic is correct. The +//! sibling `.wast` tests cover the pure fast path (no host calls); here we +//! deliberately force the slow path with a real host import. + +#![cfg(not(miri))] + +use wasmtime::component::*; +use wasmtime::{Config, Engine, Result, Store, StoreContextMut}; + +/// An engine whose stores have component-model concurrency support (and thus +/// the inline sync-to-sync adapter optimization) enabled. +fn engine() -> Engine { + let mut config = Config::new(); + config.wasm_component_model(true); + config.wasm_component_model_async(true); + Engine::new(&config).unwrap() +} + +/// A single guest-to-guest sync call whose callee forces the deferred thread by +/// calling an imported host function. The slow `exit-sync-call` path must still +/// restore the caller's context slot, and the callee's mutation must not leak. +#[tokio::test] +async fn host_call_forces_slow_path_preserves_context() -> Result<()> { + let component = r#" +(component + (import "poke" (func $poke)) + + (component $Inner + (import "poke" (func $poke)) + (core func $poke' (canon lower (func $poke))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "poke" (func $poke')) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "f'") (param i32) (result i32) + ;; Freshly entered deferred thread: context starts zeroed. + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x5678)) + ;; Force the deferred thread via a guest->host call. + (call $poke') + ;; Our context survives the force. + (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable)) + (i32.add (local.get 0) (i32.const 42)))) + (core instance $m (instantiate $M (with "" (instance + (export "poke" (func $poke')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'")))) + + (component $Outer + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $cset (i32.const 0x1234)) + (local.set $r (call $f' (i32.const 1234))) + ;; Restored after the callee forced the slow exit path. + (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable)) + (local.get $r))) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "g") (result u32) + (canon lift (core func $n "g'")))) + + (instance $inner (instantiate $Inner (with "poke" (func $poke)))) + (instance $outer (instantiate $Outer (with "f" (func $inner "f")))) + (export "g" (func $outer "g")) +) + "#; + + let engine = engine(); + let component = Component::new(&engine, component)?; + let mut store = Store::new(&engine, 0u32); + let mut linker = Linker::new(&engine); + linker + .root() + .func_wrap("poke", |mut cx: StoreContextMut, (): ()| { + *cx.data_mut() += 1; + Ok(()) + })?; + let instance = linker.instantiate_async(&mut store, &component).await?; + let g = instance.get_typed_func::<(), (u32,)>(&mut store, "g")?; + + let (result,) = g.call_async(&mut store, ()).await?; + assert_eq!(result, 1276); + assert_eq!(*store.data(), 1, "host import should have been called once"); + Ok(()) +} + +/// A three-deep guest-to-guest chain (Root -> Mid -> Leaf) where the innermost +/// callee forces. Forcing must walk and replay *both* suspended deferred frames +/// and every level's context slot must come back intact. +#[tokio::test] +async fn nested_chain_host_force_preserves_all_contexts() -> Result<()> { + let component = r#" +(component + (import "poke" (func $poke)) + + (component $Leaf + (import "poke" (func $poke)) + (core func $poke' (canon lower (func $poke))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "poke" (func $poke')) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "leaf'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x0c0ffee0)) + (call $poke') + (if (i32.ne (call $cget) (i32.const 0x0c0ffee0)) (then unreachable)) + (i32.add (local.get 0) (i32.const 1)))) + (core instance $m (instantiate $M (with "" (instance + (export "poke" (func $poke')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "leaf") (param "x" u32) (result u32) + (canon lift (core func $m "leaf'")))) + + (component $Mid + (import "leaf" (func $leaf (param "x" u32) (result u32))) + (core func $leaf' (canon lower (func $leaf))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "leaf'" (func $leaf' (param i32) (result i32))) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "mid'") (param i32) (result i32) (local $r i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x0d00d100)) + (local.set $r (call $leaf' (local.get 0))) + (if (i32.ne (call $cget) (i32.const 0x0d00d100)) (then unreachable)) + (i32.add (local.get $r) (i32.const 10)))) + (core instance $m (instantiate $M (with "" (instance + (export "leaf'" (func $leaf')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "mid") (param "x" u32) (result u32) + (canon lift (core func $m "mid'")))) + + (component $Root + (import "mid" (func $mid (param "x" u32) (result u32))) + (core func $mid' (canon lower (func $mid))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "mid'" (func $mid' (param i32) (result i32))) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "root'") (result i32) (local $r i32) + (call $cset (i32.const 0x0badf00d)) + (local.set $r (call $mid' (i32.const 100))) + (if (i32.ne (call $cget) (i32.const 0x0badf00d)) (then unreachable)) + (i32.add (local.get $r) (i32.const 1000)))) + (core instance $m (instantiate $M (with "" (instance + (export "mid'" (func $mid')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "root") (result u32) + (canon lift (core func $m "root'")))) + + (instance $leaf (instantiate $Leaf (with "poke" (func $poke)))) + (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf")))) + (instance $root (instantiate $Root (with "mid" (func $mid "mid")))) + (export "root" (func $root "root")) +) + "#; + + let engine = engine(); + let component = Component::new(&engine, component)?; + let mut store = Store::new(&engine, 0u32); + let mut linker = Linker::new(&engine); + linker + .root() + .func_wrap("poke", |mut cx: StoreContextMut, (): ()| { + *cx.data_mut() += 1; + Ok(()) + })?; + let instance = linker.instantiate_async(&mut store, &component).await?; + let root = instance.get_typed_func::<(), (u32,)>(&mut store, "root")?; + + let (result,) = root.call_async(&mut store, ()).await?; + assert_eq!(result, 1111); + assert_eq!(*store.data(), 1); + Ok(()) +} + +/// Repeated top-level calls must each set up and tear down their own deferred +/// frame independently: the callee keeps observing a freshly zeroed context and +/// the caller's context is restored every time, with no state left dangling +/// between adapter invocations. +#[tokio::test] +async fn repeated_calls_have_no_state_leak() -> Result<()> { + let component = r#" +(component + (import "poke" (func $poke)) + + (component $Inner + (import "poke" (func $poke)) + (core func $poke' (canon lower (func $poke))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "poke" (func $poke')) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "f'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (local.get 0)) + (call $poke') + (if (i32.ne (call $cget) (local.get 0)) (then unreachable)) + (i32.add (local.get 0) (i32.const 42)))) + (core instance $m (instantiate $M (with "" (instance + (export "poke" (func $poke')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'")))) + + (component $Outer + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "cget" (func $cget (result i32))) + (import "" "cset" (func $cset (param i32))) + (func (export "g'") (param i32) (result i32) (local $r i32) + (call $cset (i32.const 0x4321)) + (local.set $r (call $f' (local.get 0))) + (if (i32.ne (call $cget) (i32.const 0x4321)) (then unreachable)) + (local.get $r))) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "cget" (func $cget)) + (export "cset" (func $cset)))))) + (func (export "g") (param "x" u32) (result u32) + (canon lift (core func $n "g'")))) + + (instance $inner (instantiate $Inner (with "poke" (func $poke)))) + (instance $outer (instantiate $Outer (with "f" (func $inner "f")))) + (export "g" (func $outer "g")) +) + "#; + + let engine = engine(); + let component = Component::new(&engine, component)?; + let mut store = Store::new(&engine, 0u32); + let mut linker = Linker::new(&engine); + linker + .root() + .func_wrap("poke", |mut cx: StoreContextMut, (): ()| { + *cx.data_mut() += 1; + Ok(()) + })?; + let instance = linker.instantiate_async(&mut store, &component).await?; + let g = instance.get_typed_func::<(u32,), (u32,)>(&mut store, "g")?; + + for x in [7u32, 100, 0x10000, 1] { + let (result,) = g.call_async(&mut store, (x,)).await?; + assert_eq!(result, x + 42); + } + assert_eq!( + *store.data(), + 4, + "host import called once per top-level call" + ); + Ok(()) +} + +/// Regression test for a use-after-free in the inline sync-to-sync fast path +/// (issue #12311). +/// +/// The inline `enter-sync-call` publishes `VMStoreContext::current_thread` +/// pointing at a `VMDeferredThread` that lives in the *fused adapter's stack +/// frame*. If the callee traps, the stack unwinds and discards that frame; the +/// teardown `exit_guest_sync_call` runs only on the success path (via +/// post-return), so without a fix-up `current_thread` would be left dangling in +/// the store. +/// +/// Re-*entering* the component is gated by `may_enter`, which short-circuits on +/// `trapped()` before forcing -- but *instantiation* is not gated. Instantiating +/// another component in the same store runs `enter_guest_sync_call` -> +/// `force_current_thread`, whose `unsafe { &*ptr }` parent-chain walk would then +/// dereference the dangling deferred-thread pointer (a freed stack frame). A +/// two-deep deferred chain (Root -> Mid -> Leaf, where Leaf traps) leaves two +/// freed `VMDeferredThread` records linked by `parent`, so the walk follows a +/// `parent` pointer read out of freed stack memory. +/// +/// Before the fix this aborted/segfaulted reliably (the freed `parent` slot +/// being read as a misaligned/garbage pointer); `invoke_wasm_and_catch_traps` +/// now resets `current_thread` to the "forced" sentinel on the trap path, so the +/// later force is a cheap no-op and this instantiation completes cleanly. +#[tokio::test] +async fn trap_then_instantiate_uses_freed_deferred_thread() -> Result<()> { + // A nested guest-to-guest sync chain (Root -> Mid -> Leaf) whose innermost + // callee traps mid-flight. Each hop's fused adapter has published a + // `VMDeferredThread` in its own (now unwound) stack frame, and + // `current_thread` is left pointing at the innermost one with a `parent` + // link to the next -- two dangling records for `force_current_thread` to + // walk. + let trapping = r#" +(component + (component $Leaf + (core module $M (func (export "leaf'") (param i32) (result i32) unreachable)) + (core instance $m (instantiate $M)) + (func (export "leaf") (param "x" u32) (result u32) (canon lift (core func $m "leaf'")))) + (component $Mid + (import "leaf" (func $leaf (param "x" u32) (result u32))) + (core func $leaf' (canon lower (func $leaf))) + (core module $M + (import "" "leaf'" (func $leaf' (param i32) (result i32))) + (func (export "mid'") (param i32) (result i32) (call $leaf' (local.get 0)))) + (core instance $m (instantiate $M (with "" (instance (export "leaf'" (func $leaf')))))) + (func (export "mid") (param "x" u32) (result u32) (canon lift (core func $m "mid'")))) + (component $Root + (import "mid" (func $mid (param "x" u32) (result u32))) + (core func $mid' (canon lower (func $mid))) + (core module $M + (import "" "mid'" (func $mid' (param i32) (result i32))) + (func (export "root'") (result i32) (call $mid' (i32.const 1)))) + (core instance $m (instantiate $M (with "" (instance (export "mid'" (func $mid')))))) + (func (export "root") (result u32) (canon lift (core func $m "root'")))) + (instance $leaf (instantiate $Leaf)) + (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf")))) + (instance $root (instantiate $Root (with "mid" (func $mid "mid")))) + (export "root" (func $root "root")) +) + "#; + + // Any second component whose instantiation drives a core instance runs + // `enter_guest_sync_call` -> `force_current_thread`, which reads the dangling + // pointer left behind by the trap above. + let other = r#" +(component + (core module $m (func (export "x"))) + (core instance (instantiate $m)) +) + "#; + + let engine = engine(); + let trapping = Component::new(&engine, trapping)?; + let other = Component::new(&engine, other)?; + let mut store = Store::new(&engine, 0u32); + let linker = Linker::new(&engine); + + let instance = linker.instantiate_async(&mut store, &trapping).await?; + let root = instance.get_typed_func::<(), (u32,)>(&mut store, "root")?; + let err = root.call_async(&mut store, ()).await.unwrap_err(); + assert!( + err.downcast_ref::().is_some(), + "expected a trap, got: {err:?}" + ); + + // BUG: this instantiation forces the dangling deferred thread (use-after-free). + // With the bug fixed it instantiates cleanly and the test passes. + let _ = linker.instantiate_async(&mut store, &other).await?; + Ok(()) +} diff --git a/tests/disas/component-model/sync-adapter-calls.wat b/tests/disas/component-model/sync-adapter-calls.wat new file mode 100644 index 000000000000..f2ad71a97038 --- /dev/null +++ b/tests/disas/component-model/sync-adapter-calls.wat @@ -0,0 +1,354 @@ +;;! target = "x86_64" +;;! test = "optimize" +;;! filter = "function" +;;! flags = "-C inlining=y -Wconcurrency-support=y" + +(component + (component $A + (core module $M + (func (export "f'") (param i32) (result i32) + (i32.add (local.get 0) (i32.const 42)) + ) + ) + + (core instance $m (instantiate $M)) + + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'")) + ) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + + (core func $f' (canon lower (func $f))) + + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (func (export "g'") (result i32) + (call $f' (i32.const 1234)) + ) + ) + + (core instance $n + (instantiate $N + (with "" (instance (export "f'" (func $f')))) + ) + ) + + (func (export "g") (result u32) + (canon lift (core func $n "g'")) + ) + ) + + (instance $a (instantiate $A)) + (instance $b + (instantiate $B + (with "f" (func $a "f")) + ) + ) + + (export "g" (func $b "g")) +) +;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 268435480 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @003b jump block1 +;; +;; block1: +;; @0038 v4 = iconst.i32 42 +;; v5 = iadd.i32 v2, v4 ; v4 = 42 +;; @003b return v5 +;; } +;; +;; function u1:0(i64 vmctx, i64) -> i32 tail { +;; ss0 = explicit_slot 32, align = 8 +;; region0 = 8 "VMContext+0x8" +;; region1 = 268435480 "VMStoreContext+0x18" +;; region2 = 72 "VMContext+0x48" +;; region3 = 200 "VMContext+0xc8" +;; region4 = 1610612736 "PublicGlobal" +;; region5 = 104 "VMContext+0x68" +;; region6 = 88 "VMContext+0x58" +;; region7 = 224 "VMContext+0xe0" +;; region8 = 136 "VMContext+0x88" +;; region9 = 268435592 "VMStoreContext+0x88" +;; region10 = 2952790016 "VMDeferredThread+0x0" +;; region11 = 2952790024 "VMDeferredThread+0x8" +;; region12 = 2952790028 "VMDeferredThread+0xc" +;; region13 = 2952790032 "VMDeferredThread+0x10" +;; region14 = 2952790036 "VMDeferredThread+0x14" +;; region15 = 2952790040 "VMDeferredThread+0x18" +;; region16 = 176 "VMContext+0xb0" +;; region17 = 168 "VMContext+0xa8" +;; region18 = 152 "VMContext+0x98" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move region0 gv3+8 +;; gv5 = load.i64 notrap aligned region1 gv4+24 +;; gv6 = vmctx +;; gv7 = load.i64 notrap aligned readonly can_move region0 gv6+8 +;; gv8 = load.i64 notrap aligned region1 gv7+24 +;; sig0 = (i64 vmctx, i64, i32) -> i32 tail +;; sig1 = (i64 vmctx, i64, i32) tail +;; sig2 = (i64 vmctx, i64, i32, i32, i32) tail +;; sig3 = (i64 vmctx, i64, i32) -> i32 tail +;; sig4 = (i64 vmctx, i64) tail +;; fn0 = colocated u2:0 sig0 +;; fn1 = colocated u0:0 sig3 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @00ee jump block2 +;; +;; block2: +;; jump block6 +;; +;; block8(v9: i64): +;; jump block5 +;; +;; block6: +;; @00ee v4 = load.i64 notrap aligned readonly can_move region2 v0+72 +;; v14 = load.i64 notrap aligned readonly can_move region3 v4+200 +;; v15 = load.i32 notrap aligned region4 v14 +;; v16 = iconst.i32 1 +;; v17 = band v15, v16 ; v16 = 1 +;; v13 = iconst.i32 0 +;; v19 = icmp eq v17, v13 ; v13 = 0 +;; brif v19, block9, block10 +;; +;; block9: +;; v23 = load.i64 notrap aligned readonly can_move region6 v4+88 +;; v22 = load.i64 notrap aligned readonly can_move region5 v4+104 +;; v21 = iconst.i32 23 +;; try_call_indirect v23(v22, v4, v21), sig1, block11, [ context v4, default: block8(exn0) ] ; v21 = 23 +;; +;; block11: +;; trap user12 +;; +;; block10: +;; v28 = load.i64 notrap aligned readonly can_move region7 v4+224 +;; v29 = load.i32 notrap aligned region4 v28 +;; v87 = iconst.i32 0 +;; store notrap aligned region4 v87, v28 ; v87 = 0 +;; v37 = load.i64 notrap aligned readonly can_move region0 v4+8 +;; v38 = load.i64 notrap aligned region9 v37+136 +;; v36 = stack_addr.i64 ss0 +;; store notrap aligned region10 v38, v36 +;; v32 = iconst.i32 2 +;; store notrap aligned region11 v32, v36+8 ; v32 = 2 +;; store notrap aligned region12 v87, v36+12 ; v87 = 0 +;; v88 = iconst.i32 1 +;; store notrap aligned region13 v88, v36+16 ; v88 = 1 +;; v39 = load.i32 notrap aligned v37+128 +;; store notrap aligned region14 v39, v36+20 +;; store notrap aligned v87, v37+128 ; v87 = 0 +;; v41 = load.i32 notrap aligned v37+132 +;; store notrap aligned region15 v41, v36+24 +;; store notrap aligned v87, v37+132 ; v87 = 0 +;; store notrap aligned region9 v36, v37+136 +;; v43 = load.i64 notrap aligned readonly can_move region16 v4+176 +;; v44 = load.i32 notrap aligned region4 v43 +;; v45 = iconst.i32 -2 +;; v46 = band v44, v45 ; v45 = -2 +;; store notrap aligned region4 v46, v43 +;; v89 = bor v44, v88 ; v88 = 1 +;; store notrap aligned region4 v89, v43 +;; jump block17 +;; +;; block17: +;; jump block18 +;; +;; block18: +;; jump block12 +;; +;; block12: +;; jump block13 +;; +;; block13: +;; v61 = load.i64 notrap aligned region10 v36 +;; store notrap aligned region9 v61, v37+136 +;; v62 = load.i32 notrap aligned region14 v36+20 +;; store notrap aligned v62, v37+128 +;; v63 = load.i32 notrap aligned region15 v36+24 +;; store notrap aligned v63, v37+132 +;; jump block15 +;; +;; block15: +;; v65 = load.i32 notrap aligned region4 v14 +;; v90 = iconst.i32 -2 +;; v91 = band v65, v90 ; v90 = -2 +;; store notrap aligned region4 v91, v14 +;; v92 = iconst.i32 1 +;; v93 = bor v65, v92 ; v92 = 1 +;; store notrap aligned region4 v93, v14 +;; store.i32 notrap aligned region4 v29, v28 +;; jump block7 +;; +;; block7: +;; jump block4 +;; +;; block5: +;; v94 = load.i64 notrap aligned readonly can_move region6 v4+88 +;; v95 = load.i64 notrap aligned readonly can_move region5 v4+104 +;; v25 = iconst.i32 49 +;; call_indirect sig1, v94(v95, v4, v25) ; v25 = 49 +;; trap user12 +;; +;; block4: +;; jump block3 +;; +;; block3: +;; jump block19 +;; +;; block19: +;; @00f0 jump block1 +;; +;; block1: +;; v78 = iconst.i32 1276 +;; @00f0 return v78 ; v78 = 1276 +;; } +;; +;; function u2:0(i64 vmctx, i64, i32) -> i32 tail { +;; ss0 = explicit_slot 32, align = 8 +;; region0 = 8 "VMContext+0x8" +;; region1 = 268435480 "VMStoreContext+0x18" +;; region2 = 200 "VMContext+0xc8" +;; region3 = 1610612736 "PublicGlobal" +;; region4 = 104 "VMContext+0x68" +;; region5 = 88 "VMContext+0x58" +;; region6 = 224 "VMContext+0xe0" +;; region7 = 136 "VMContext+0x88" +;; region8 = 268435592 "VMStoreContext+0x88" +;; region9 = 2952790016 "VMDeferredThread+0x0" +;; region10 = 2952790024 "VMDeferredThread+0x8" +;; region11 = 2952790028 "VMDeferredThread+0xc" +;; region12 = 2952790032 "VMDeferredThread+0x10" +;; region13 = 2952790036 "VMDeferredThread+0x14" +;; region14 = 2952790040 "VMDeferredThread+0x18" +;; region15 = 176 "VMContext+0xb0" +;; region16 = 72 "VMContext+0x48" +;; region17 = 168 "VMContext+0xa8" +;; region18 = 152 "VMContext+0x98" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move region0 gv3+8 +;; gv5 = load.i64 notrap aligned region1 gv4+24 +;; sig0 = (i64 vmctx, i64, i32) tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) tail +;; sig2 = (i64 vmctx, i64, i32) -> i32 tail +;; sig3 = (i64 vmctx, i64) tail +;; fn0 = colocated u0:0 sig2 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @00cf jump block4 +;; +;; block6(v7: i64): +;; @00cf jump block3 +;; +;; block4: +;; @00d4 v9 = load.i64 notrap aligned readonly can_move region2 v0+200 +;; @00d4 v10 = load.i32 notrap aligned region3 v9 +;; @00d6 v11 = iconst.i32 1 +;; @00d8 v12 = band v10, v11 ; v11 = 1 +;; @00c9 v4 = iconst.i32 0 +;; @00d9 v14 = icmp eq v12, v4 ; v4 = 0 +;; @00da brif v14, block7, block8 +;; +;; block7: +;; @00de v18 = load.i64 notrap aligned readonly can_move region5 v0+88 +;; @00de v17 = load.i64 notrap aligned readonly can_move region4 v0+104 +;; @00dc v16 = iconst.i32 23 +;; @00de try_call_indirect v18(v17, v0, v16), sig0, block9, [ context v0, default: block6(exn0) ] ; v16 = 23 +;; +;; block9: +;; @00e0 trap user12 +;; +;; block8: +;; @00e2 v19 = load.i64 notrap aligned readonly can_move region6 v0+224 +;; @00e2 v20 = load.i32 notrap aligned region3 v19 +;; v79 = iconst.i32 0 +;; @00e8 store notrap aligned region3 v79, v19 ; v79 = 0 +;; @00f0 v28 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @00f0 v29 = load.i64 notrap aligned region8 v28+136 +;; @00f0 v27 = stack_addr.i64 ss0 +;; @00f0 store notrap aligned region9 v29, v27 +;; @00ea v23 = iconst.i32 2 +;; @00f0 store notrap aligned region10 v23, v27+8 ; v23 = 2 +;; @00f0 store notrap aligned region11 v79, v27+12 ; v79 = 0 +;; v80 = iconst.i32 1 +;; @00f0 store notrap aligned region12 v80, v27+16 ; v80 = 1 +;; @00f0 v30 = load.i32 notrap aligned v28+128 +;; @00f0 store notrap aligned region13 v30, v27+20 +;; @00f0 store notrap aligned v79, v28+128 ; v79 = 0 +;; @00f0 v32 = load.i32 notrap aligned v28+132 +;; @00f0 store notrap aligned region14 v32, v27+24 +;; @00f0 store notrap aligned v79, v28+132 ; v79 = 0 +;; @00f0 store notrap aligned region8 v27, v28+136 +;; @00f2 v34 = load.i64 notrap aligned readonly can_move region15 v0+176 +;; @00f2 v35 = load.i32 notrap aligned region3 v34 +;; @00f4 v36 = iconst.i32 -2 +;; @00f6 v37 = band v35, v36 ; v36 = -2 +;; @00f7 store notrap aligned region3 v37, v34 +;; v81 = bor v35, v80 ; v80 = 1 +;; @0100 store notrap aligned region3 v81, v34 +;; @0102 jump block15 +;; +;; block15: +;; jump block16 +;; +;; block16: +;; jump block10 +;; +;; block10: +;; @0106 jump block11 +;; +;; block11: +;; @0106 v51 = load.i64 notrap aligned region9 v27 +;; @0106 store notrap aligned region8 v51, v28+136 +;; @0106 v52 = load.i32 notrap aligned region13 v27+20 +;; @0106 store notrap aligned v52, v28+128 +;; @0106 v53 = load.i32 notrap aligned region14 v27+24 +;; @0106 store notrap aligned v53, v28+132 +;; @0106 jump block13 +;; +;; block13: +;; @0108 v56 = load.i32 notrap aligned region3 v9 +;; v82 = iconst.i32 -2 +;; v83 = band v56, v82 ; v82 = -2 +;; @010d store notrap aligned region3 v83, v9 +;; v84 = iconst.i32 1 +;; v85 = bor v56, v84 ; v84 = 1 +;; @0116 store notrap aligned region3 v85, v9 +;; @011a store.i32 notrap aligned region3 v20, v19 +;; @011c jump block5 +;; +;; block5: +;; @011d jump block2 +;; +;; block3: +;; v86 = load.i64 notrap aligned readonly can_move region5 v0+88 +;; v87 = load.i64 notrap aligned readonly can_move region4 v0+104 +;; @0120 v68 = iconst.i32 49 +;; @0122 call_indirect sig0, v86(v87, v0, v68) ; v68 = 49 +;; @0124 trap user12 +;; +;; block2: +;; @0126 jump block1 +;; +;; block1: +;; v72 = iconst.i32 42 +;; v73 = iadd.i32 v2, v72 ; v72 = 42 +;; @0126 return v73 +;; } diff --git a/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast b/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast new file mode 100644 index 000000000000..862415b970fd --- /dev/null +++ b/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast @@ -0,0 +1,158 @@ +;;! component_model_async = true +;;! component_model_more_async_builtins = true +;;! component_model_threading = true + +;; Like sync-call-context.wast, but drives *both* component-context slots (0 and +;; 1) at once. The inline `enter-sync-call`/`exit-sync-call` save, zero, and +;; restore every slot in a loop over `NUM_COMPONENT_CONTEXT_SLOTS`, and the +;; per-slot frame offset `vmdeferred_thread_saved_context(i)` is only spot +;; checked for i == 0 by the `deferred_thread_field_offsets` unit test. Using +;; distinct values in slots 0 and 1 catches a wrong stride / slot-swap for the +;; second slot, across both the fast path and the forced slow path. +;; +;; Slot 1 requires `component-model-threading`. + +;; --- Test A: both slots, fast path. --- +(component + (component $A + (core func $get0 (canon context.get i32 0)) + (core func $set0 (canon context.set i32 0)) + (core func $get1 (canon context.get i32 1)) + (core func $set1 (canon context.set i32 1)) + (core module $M + (import "" "get0" (func $get0 (result i32))) + (import "" "set0" (func $set0 (param i32))) + (import "" "get1" (func $get1 (result i32))) + (import "" "set1" (func $set1 (param i32))) + (func (export "f'") (param i32) (result i32) + ;; Fresh thread: both slots zero. + (if (i32.ne (call $get0) (i32.const 0)) (then unreachable)) + (if (i32.ne (call $get1) (i32.const 0)) (then unreachable)) + (call $set0 (i32.const 0xAAAA1111)) + (call $set1 (i32.const 0xBBBB2222)) + ;; The two slots must not alias each other. + (if (i32.ne (call $get0) (i32.const 0xAAAA1111)) (then unreachable)) + (if (i32.ne (call $get1) (i32.const 0xBBBB2222)) (then unreachable)) + (i32.add (local.get 0) (i32.const 42)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "get0" (func $get0)) (export "set0" (func $set0)) + (export "get1" (func $get1)) (export "set1" (func $set1)) + )))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'"))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $get0 (canon context.get i32 0)) + (core func $set0 (canon context.set i32 0)) + (core func $get1 (canon context.get i32 1)) + (core func $set1 (canon context.set i32 1)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "get0" (func $get0 (result i32))) + (import "" "set0" (func $set0 (param i32))) + (import "" "get1" (func $get1 (result i32))) + (import "" "set1" (func $set1 (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $set0 (i32.const 0x11110000)) + (call $set1 (i32.const 0x22220000)) + (local.set $r (call $f' (i32.const 1234))) + ;; Both of our slots restored, independently and unswapped. + (if (i32.ne (call $get0) (i32.const 0x11110000)) (then unreachable)) + (if (i32.ne (call $get1) (i32.const 0x22220000)) (then unreachable)) + (local.get $r) + ) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "get0" (func $get0)) (export "set0" (func $set0)) + (export "get1" (func $get1)) (export "set1" (func $set1)) + )))) + (func (export "g") (result u32) + (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 1276)) + +;; --- Test B: both slots, forced slow path. --- +(component + (component $A + (core func $get0 (canon context.get i32 0)) + (core func $set0 (canon context.set i32 0)) + (core func $get1 (canon context.get i32 1)) + (core func $set1 (canon context.set i32 1)) + (core func $bpinc (canon backpressure.inc)) + (core func $bpdec (canon backpressure.dec)) + (core module $M + (import "" "get0" (func $get0 (result i32))) + (import "" "set0" (func $set0 (param i32))) + (import "" "get1" (func $get1 (result i32))) + (import "" "set1" (func $set1 (param i32))) + (import "" "backpressure.inc" (func $bpinc)) + (import "" "backpressure.dec" (func $bpdec)) + (func (export "f'") (param i32) (result i32) + (if (i32.ne (call $get0) (i32.const 0)) (then unreachable)) + (if (i32.ne (call $get1) (i32.const 0)) (then unreachable)) + (call $set0 (i32.const 0xAAAA1111)) + (call $set1 (i32.const 0xBBBB2222)) + (call $bpinc) (call $bpdec) + ;; Both slots survive the force. + (if (i32.ne (call $get0) (i32.const 0xAAAA1111)) (then unreachable)) + (if (i32.ne (call $get1) (i32.const 0xBBBB2222)) (then unreachable)) + (i32.add (local.get 0) (i32.const 42)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "get0" (func $get0)) (export "set0" (func $set0)) + (export "get1" (func $get1)) (export "set1" (func $set1)) + (export "backpressure.inc" (func $bpinc)) + (export "backpressure.dec" (func $bpdec)) + )))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'"))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $get0 (canon context.get i32 0)) + (core func $set0 (canon context.set i32 0)) + (core func $get1 (canon context.get i32 1)) + (core func $set1 (canon context.set i32 1)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "get0" (func $get0 (result i32))) + (import "" "set0" (func $set0 (param i32))) + (import "" "get1" (func $get1 (result i32))) + (import "" "set1" (func $set1 (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $set0 (i32.const 0x11110000)) + (call $set1 (i32.const 0x22220000)) + (local.set $r (call $f' (i32.const 1234))) + (if (i32.ne (call $get0) (i32.const 0x11110000)) (then unreachable)) + (if (i32.ne (call $get1) (i32.const 0x22220000)) (then unreachable)) + (local.get $r) + ) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "get0" (func $get0)) (export "set0" (func $set0)) + (export "get1" (func $get1)) (export "set1" (func $set1)) + )))) + (func (export "g") (result u32) + (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 1276)) diff --git a/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast b/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast new file mode 100644 index 000000000000..3bd5fc1b6359 --- /dev/null +++ b/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast @@ -0,0 +1,56 @@ +;;! component_model_async = true +;;! component_model_more_async_builtins = true + +;; A guest-to-guest sync call whose callee traps part way through (issue +;; #12311). The fused adapter's inline `enter-sync-call` has already published +;; an on-stack `VMDeferredThread`, but the inline `exit-sync-call` never runs: +;; the trap unwinds through the adapter's exception landing pad instead, which +;; must materialize/tear down the deferred thread (via the cleanup libcall's +;; `force_current_thread`) without reading freed stack memory. We only assert +;; the trap here -- a component trap poisons the store for further entry -- so +;; this is the sole directive in its own file. +;; +;; The callee first establishes some context-slot state so the deferred thread +;; is non-trivial at the point of the trap. +(component + (component $A + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "context.set" (func $cset (param i32))) + (func (export "f'") (param i32) (result i32) + (call $cset (i32.const 0x5678)) + unreachable + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.set" (func $cset)) + )))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'"))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $cset (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "g'") (result i32) + (call $cset (i32.const 0x1234)) + (call $f' (i32.const 1234)) + ) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "context.set" (func $cset)) + )))) + (func (export "g") (result u32) + (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_trap (invoke "g") "wasm `unreachable` instruction executed") diff --git a/tests/misc_testsuite/component-model/async/sync-call-context.wast b/tests/misc_testsuite/component-model/async/sync-call-context.wast new file mode 100644 index 000000000000..e7e009538552 --- /dev/null +++ b/tests/misc_testsuite/component-model/async/sync-call-context.wast @@ -0,0 +1,604 @@ +;;! component_model_async = true +;;! component_model_more_async_builtins = true + +;; Runtime tests for the guest-to-guest sync-call fast path (issue #12311). +;; +;; When concurrency support is enabled, a fused sync-to-sync adapter's +;; `enter-sync-call`/`exit-sync-call` intrinsics are lowered *inline* by the +;; compiler instead of calling the out-of-line libcalls: `enter` pushes an +;; on-stack `VMDeferredThread`, saving the caller's `context.{get,set}` slots +;; and zeroing them for the freshly-entered (deferred) callee thread; the +;; fast-path `exit` pops it and restores the caller's slots. If host code reads +;; the current thread mid-call (any fallible guest->host libcall, e.g. +;; `backpressure.{inc,dec}`) the deferred thread is "forced" into a real one and +;; `exit` instead takes the out-of-line slow path. +;; +;; These tests use `context.{get,set}` (slot 0) as a guest-observable witness +;; for the save / zero / restore / replay logic across the fast path, the +;; forced slow path, and nested chains. Each component returns an arithmetic +;; value so the result also witnesses correct value flow through the adapter. + +;; --------------------------------------------------------------------------- +;; Test 1: single guest-to-guest sync call, fast path (no forcing). +;; +;; $B sets its context, calls $A, and checks its context is restored. $A +;; (the deferred callee) must observe a freshly-zeroed context. +;; --------------------------------------------------------------------------- +(component + (component $A + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "f'") (param i32) (result i32) + ;; A is a freshly-entered (deferred) thread: its context starts at 0. + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x5678)) + (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable)) + (i32.add (local.get 0) (i32.const 42)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'"))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $cset (i32.const 0x1234)) + (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable)) + (local.set $r (call $f' (i32.const 1234))) + ;; The callee's context mutation must NOT leak: ours is restored. + (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable)) + (local.get $r) + ) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "g") (result u32) + (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 1276)) + +;; --------------------------------------------------------------------------- +;; Test 2: single guest-to-guest sync call, forced slow path. +;; +;; Same as test 1, but the callee $A makes a fallible guest->host libcall +;; (`backpressure.inc`/`dec`, net zero) which forces the deferred thread into a +;; real one. The matching `exit-sync-call` therefore takes the out-of-line slow +;; path, and `force_current_thread` must still preserve/restore both threads' +;; context slots. +;; --------------------------------------------------------------------------- +(component + (component $A + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core func $bpinc (canon backpressure.inc)) + (core func $bpdec (canon backpressure.dec)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (import "" "backpressure.inc" (func $bpinc)) + (import "" "backpressure.dec" (func $bpdec)) + (func (export "f'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x5678)) + ;; Force promotion of the deferred thread mid-call. + (call $bpinc) + (call $bpdec) + ;; Our context must survive the force. + (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable)) + (i32.add (local.get 0) (i32.const 42)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + (export "backpressure.inc" (func $bpinc)) + (export "backpressure.dec" (func $bpdec)) + )))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'"))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $cset (i32.const 0x1234)) + (local.set $r (call $f' (i32.const 1234))) + ;; Restored even though the callee forced the slow exit path. + (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable)) + (local.get $r) + ) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "g") (result u32) + (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 1276)) + +;; --------------------------------------------------------------------------- +;; Test 3: nested A->B->C sync-call chain, fast path. +;; +;; $Root calls $Mid calls $Leaf, each through its own fused adapter. Each level +;; must see a fresh context, and each caller's context must be restored after +;; its callee returns (two deferred frames active at the deepest point). +;; --------------------------------------------------------------------------- +(component + (component $Leaf + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "leaf'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x99aabbcc)) + (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable)) + (i32.add (local.get 0) (i32.const 1)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "leaf") (param "x" u32) (result u32) + (canon lift (core func $m "leaf'"))) + ) + + (component $Mid + (import "leaf" (func $leaf (param "x" u32) (result u32))) + (core func $leaf' (canon lower (func $leaf))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "leaf'" (func $leaf' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "mid'") (param i32) (result i32) (local $r i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x55667788)) + (local.set $r (call $leaf' (local.get 0))) + (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable)) + (i32.add (local.get $r) (i32.const 10)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "leaf'" (func $leaf')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "mid") (param "x" u32) (result u32) + (canon lift (core func $m "mid'"))) + ) + + (component $Root + (import "mid" (func $mid (param "x" u32) (result u32))) + (core func $mid' (canon lower (func $mid))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "mid'" (func $mid' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "root'") (result i32) (local $r i32) + (call $cset (i32.const 0x11223344)) + (local.set $r (call $mid' (i32.const 100))) + (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable)) + (i32.add (local.get $r) (i32.const 1000)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "mid'" (func $mid')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "root") (result u32) + (canon lift (core func $m "root'"))) + ) + + (instance $leaf (instantiate $Leaf)) + (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf")))) + (instance $root (instantiate $Root (with "mid" (func $mid "mid")))) + (export "root" (func $root "root")) +) +(assert_return (invoke "root") (u32.const 1111)) + +;; --------------------------------------------------------------------------- +;; Test 4: nested chain, forced at the deepest level. +;; +;; As test 3, but $Leaf forces the current thread mid-call. At that point two +;; deferred frames ($Mid and $Leaf) are linked above the materialized $Root +;; base; `force_current_thread` must walk and replay both, and both adapter +;; exits then take the slow path. Every level's context must still be restored. +;; --------------------------------------------------------------------------- +(component + (component $Leaf + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core func $bpinc (canon backpressure.inc)) + (core func $bpdec (canon backpressure.dec)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (import "" "backpressure.inc" (func $bpinc)) + (import "" "backpressure.dec" (func $bpdec)) + (func (export "leaf'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x99aabbcc)) + (call $bpinc) + (call $bpdec) + (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable)) + (i32.add (local.get 0) (i32.const 1)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + (export "backpressure.inc" (func $bpinc)) + (export "backpressure.dec" (func $bpdec)) + )))) + (func (export "leaf") (param "x" u32) (result u32) + (canon lift (core func $m "leaf'"))) + ) + + (component $Mid + (import "leaf" (func $leaf (param "x" u32) (result u32))) + (core func $leaf' (canon lower (func $leaf))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "leaf'" (func $leaf' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "mid'") (param i32) (result i32) (local $r i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x55667788)) + (local.set $r (call $leaf' (local.get 0))) + ;; Restored after the (forced) nested call. + (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable)) + (i32.add (local.get $r) (i32.const 10)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "leaf'" (func $leaf')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "mid") (param "x" u32) (result u32) + (canon lift (core func $m "mid'"))) + ) + + (component $Root + (import "mid" (func $mid (param "x" u32) (result u32))) + (core func $mid' (canon lower (func $mid))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "mid'" (func $mid' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "root'") (result i32) (local $r i32) + (call $cset (i32.const 0x11223344)) + (local.set $r (call $mid' (i32.const 100))) + (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable)) + (i32.add (local.get $r) (i32.const 1000)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "mid'" (func $mid')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "root") (result u32) + (canon lift (core func $m "root'"))) + ) + + (instance $leaf (instantiate $Leaf)) + (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf")))) + (instance $root (instantiate $Root (with "mid" (func $mid "mid")))) + (export "root" (func $root "root")) +) +(assert_return (invoke "root") (u32.const 1111)) + +;; --------------------------------------------------------------------------- +;; Test 5: forcing at an intermediate level, then a deeper sync call. +;; +;; $Root calls $Mid; $Mid forces (materializing its thread) *before* calling +;; $Leaf. The $Mid->$Leaf adapter then pushes a fresh deferred frame on top of +;; the now-forced thread, so $Leaf's exit takes the fast path while $Mid's exit +;; takes the slow path. Contexts must remain correct across the mix. +;; --------------------------------------------------------------------------- +(component + (component $Leaf + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "leaf'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x99aabbcc)) + (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable)) + (i32.add (local.get 0) (i32.const 1)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "leaf") (param "x" u32) (result u32) + (canon lift (core func $m "leaf'"))) + ) + + (component $Mid + (import "leaf" (func $leaf (param "x" u32) (result u32))) + (core func $leaf' (canon lower (func $leaf))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core func $bpinc (canon backpressure.inc)) + (core func $bpdec (canon backpressure.dec)) + (core module $M + (import "" "leaf'" (func $leaf' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (import "" "backpressure.inc" (func $bpinc)) + (import "" "backpressure.dec" (func $bpdec)) + (func (export "mid'") (param i32) (result i32) (local $r i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x55667788)) + ;; Force *before* descending into the leaf. + (call $bpinc) + (call $bpdec) + (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable)) + (local.set $r (call $leaf' (local.get 0))) + (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable)) + (i32.add (local.get $r) (i32.const 10)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "leaf'" (func $leaf')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + (export "backpressure.inc" (func $bpinc)) + (export "backpressure.dec" (func $bpdec)) + )))) + (func (export "mid") (param "x" u32) (result u32) + (canon lift (core func $m "mid'"))) + ) + + (component $Root + (import "mid" (func $mid (param "x" u32) (result u32))) + (core func $mid' (canon lower (func $mid))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "mid'" (func $mid' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "root'") (result i32) (local $r i32) + (call $cset (i32.const 0x11223344)) + (local.set $r (call $mid' (i32.const 100))) + (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable)) + (i32.add (local.get $r) (i32.const 1000)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "mid'" (func $mid')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "root") (result u32) + (canon lift (core func $m "root'"))) + ) + + (instance $leaf (instantiate $Leaf)) + (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf")))) + (instance $root (instantiate $Root (with "mid" (func $mid "mid")))) + (export "root" (func $root "root")) +) +(assert_return (invoke "root") (u32.const 1111)) + +;; --------------------------------------------------------------------------- +;; Test 6: repeated sync calls from the same caller. +;; +;; $B calls $A twice in a row. Each call must independently push/pop its own +;; deferred frame: the second callee must still observe a freshly-zeroed +;; context, and the caller's context must be restored after each call (no state +;; left dangling between adapter invocations). +;; --------------------------------------------------------------------------- +(component + (component $A + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "f'") (param i32) (result i32) + ;; Each fresh entry must zero the context regardless of prior calls. + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.add (local.get 0) (i32.const 0x10000))) + (i32.add (local.get 0) (i32.const 42)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'"))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $cset (i32.const 0x1234)) + (local.set $r (call $f' (i32.const 1))) + (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable)) + (local.set $r (i32.add (local.get $r) (call $f' (i32.const 2)))) + ;; Still restored after the second call. + (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable)) + (local.get $r) + ) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "g") (result u32) + (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +;; (1 + 42) + (2 + 42) = 87 +(assert_return (invoke "g") (u32.const 87)) + +;; --------------------------------------------------------------------------- +;; Test 7: nested chain forced at *two* levels (re-forcing). +;; +;; $Mid forces before descending, materializing its thread; the $Mid->$Leaf +;; adapter then pushes a fresh deferred frame whose parent is already forced, +;; and $Leaf forces again. This exercises `force_current_thread` walking a +;; single deferred frame that sits directly on a forced base, distinct from the +;; two-frame walk in test 4. +;; --------------------------------------------------------------------------- +(component + (component $Leaf + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core func $bpinc (canon backpressure.inc)) + (core func $bpdec (canon backpressure.dec)) + (core module $M + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (import "" "backpressure.inc" (func $bpinc)) + (import "" "backpressure.dec" (func $bpdec)) + (func (export "leaf'") (param i32) (result i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x99aabbcc)) + (call $bpinc) (call $bpdec) + (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable)) + (i32.add (local.get 0) (i32.const 1)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + (export "backpressure.inc" (func $bpinc)) + (export "backpressure.dec" (func $bpdec)) + )))) + (func (export "leaf") (param "x" u32) (result u32) + (canon lift (core func $m "leaf'"))) + ) + + (component $Mid + (import "leaf" (func $leaf (param "x" u32) (result u32))) + (core func $leaf' (canon lower (func $leaf))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core func $bpinc (canon backpressure.inc)) + (core func $bpdec (canon backpressure.dec)) + (core module $M + (import "" "leaf'" (func $leaf' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (import "" "backpressure.inc" (func $bpinc)) + (import "" "backpressure.dec" (func $bpdec)) + (func (export "mid'") (param i32) (result i32) (local $r i32) + (if (i32.ne (call $cget) (i32.const 0)) (then unreachable)) + (call $cset (i32.const 0x55667788)) + (call $bpinc) (call $bpdec) + (local.set $r (call $leaf' (local.get 0))) + (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable)) + (i32.add (local.get $r) (i32.const 10)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "leaf'" (func $leaf')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + (export "backpressure.inc" (func $bpinc)) + (export "backpressure.dec" (func $bpdec)) + )))) + (func (export "mid") (param "x" u32) (result u32) + (canon lift (core func $m "mid'"))) + ) + + (component $Root + (import "mid" (func $mid (param "x" u32) (result u32))) + (core func $mid' (canon lower (func $mid))) + (core func $cget (canon context.get i32 0)) + (core func $cset (canon context.set i32 0)) + (core module $M + (import "" "mid'" (func $mid' (param i32) (result i32))) + (import "" "context.get" (func $cget (result i32))) + (import "" "context.set" (func $cset (param i32))) + (func (export "root'") (result i32) (local $r i32) + (call $cset (i32.const 0x11223344)) + (local.set $r (call $mid' (i32.const 100))) + (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable)) + (i32.add (local.get $r) (i32.const 1000)) + ) + ) + (core instance $m (instantiate $M (with "" (instance + (export "mid'" (func $mid')) + (export "context.get" (func $cget)) + (export "context.set" (func $cset)) + )))) + (func (export "root") (result u32) + (canon lift (core func $m "root'"))) + ) + + (instance $leaf (instantiate $Leaf)) + (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf")))) + (instance $root (instantiate $Root (with "mid" (func $mid "mid")))) + (export "root" (func $root "root")) +) +(assert_return (invoke "root") (u32.const 1111))