diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c82fb122c..1f248c3e61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed several underallocation issues that could trigger data corruption on `binary:replace`, `zlib:compress` and bsd socket recv code. - Fixed a bug where `catch` would raise on regular atom results - Fixed ESP32 socket driver holding the global socket-list lock across blocking TCP connects, leaking the port on connect failure, losing concurrent `accept` waiters, leaking `netbuf` on receive error paths, and a recycled-`netconn` race between socket close and the event handler +- Fixed a bug where AtomVM could only have 256 loaded modules ## [0.7.0-alpha.1] - 2026-04-06 diff --git a/libs/jit/src/CMakeLists.txt b/libs/jit/src/CMakeLists.txt index 375a79f5ae..bca9324278 100644 --- a/libs/jit/src/CMakeLists.txt +++ b/libs/jit/src/CMakeLists.txt @@ -122,6 +122,16 @@ foreach(riscv_module jit_riscv32 jit_riscv64) ) endforeach() +# The register-based backends include the shared native-register bookkeeping in +# jit_backend_regs_impl.hrl, so changes to that header must force recompilation. +foreach(regs_module jit_x86_64 jit_aarch64 jit_arm32 jit_armv6m jit_xtensa jit_riscv32 jit_riscv64) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/beams/${regs_module}.beam + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/jit_backend_regs_impl.hrl + APPEND + ) +endforeach() + set(JIT_VERSION ${ATOMVM_BASE_VERSION}) install( diff --git a/libs/jit/src/jit.erl b/libs/jit/src/jit.erl index efd301a97e..821c32fd83 100644 --- a/libs/jit/src/jit.erl +++ b/libs/jit/src/jit.erl @@ -272,7 +272,8 @@ first_pass(<>, MMod, MSt0, #state{tail_cache = TC} false -> Offset0 = MMod:offset(MSt0), MSt1 = MMod:move_to_cp(MSt0, {y_reg, NWords}), - MSt2 = MMod:increment_sp(MSt1, NWords + 1), + % The saved cp occupies one stack slot on 64-bit, two on 32-bit. + MSt2 = MMod:increment_sp(MSt1, NWords + (8 div MMod:word_size())), TailCacheKey1 = {op_call_only, Label}, case tail_cache_find(TailCacheKey1, TC) of false -> @@ -443,25 +444,48 @@ first_pass(<>, MMod, MSt0, State0) -> first_pass(<>, MMod, MSt0, #state{tail_cache = TC} = State0) -> ?ASSERT_ALL_NATIVE_FREE(MSt0), ?TRACE("OP_RETURN\n", []), - % Optimized return: check if returning within same module - {MSt1, CpReg0} = MMod:move_to_native_register(MSt0, cp), - {MSt2, ModuleIndexReg} = MMod:get_module_index(MSt1), - % Extract module index from cp (upper 8 bits: cp >> 24) - {MSt3, CpReg1} = MMod:shift_right(MSt2, CpReg0, 24), - % Compare extracted module index with current module index - MSt4 = MMod:if_block( - MSt3, - {{free, CpReg1}, '==', {free, ModuleIndexReg}}, - % Same module: fast intra-module return - fun(BSt0) -> - % Mask to get lower 24 bits and shift right by 2 for offset - {BSt1, CpReg0} = MMod:and_(BSt0, {free, CpReg0}, 16#FFFFFF), - {BSt3, CPReg1} = MMod:shift_right(BSt1, {free, CpReg0}, 2), - % Jump to continuation (this is a tail call) - MMod:jump_to_continuation(BSt3, {free, CPReg1}) - end - ), - MSt5 = MMod:free_native_registers(MSt4, [CpReg0]), + % Optimized return: check if returning within the same module, in which case + % we jump directly to the continuation rather than going through PRIM_RETURN. + MSt5 = + case MMod:word_size() of + 8 -> + % 64-bit: cp packs (module_index << 24) | (offset << 2) in one word. + {MSt1, CpReg0} = MMod:move_to_native_register(MSt0, cp), + {MSt2, ModuleIndexReg} = MMod:get_module_index(MSt1), + % Extract module index from cp (upper 8 bits: cp >> 24) + {MSt3, CpReg1} = MMod:shift_right(MSt2, CpReg0, 24), + % Compare extracted module index with current module index + MSt4 = MMod:if_block( + MSt3, + {{free, CpReg1}, '==', {free, ModuleIndexReg}}, + % Same module: fast intra-module return + fun(BSt0) -> + % Mask to get lower 24 bits and shift right by 2 for offset + {BSt1, CpReg0} = MMod:and_(BSt0, {free, CpReg0}, 16#FFFFFF), + {BSt3, CPReg1} = MMod:shift_right(BSt1, {free, CpReg0}, 2), + % Jump to continuation (this is a tail call) + MMod:jump_to_continuation(BSt3, {free, CPReg1}) + end + ), + MMod:free_native_registers(MSt4, [CpReg0]); + 4 -> + % 32-bit: cp spans two words, the Module pointer (?CP_MODULE) and + % the offset << 2 (?CP_OFFSET). Compare the saved Module pointer + % with the current module pointer (jit_state->module). + {MSt1, CpModReg} = MMod:get_cp_module(MSt0), + {MSt2, CurModReg} = MMod:get_module(MSt1), + MMod:if_block( + MSt2, + {{free, CpModReg}, '==', {free, CurModReg}}, + % Same module: fast intra-module return + fun(BSt0) -> + {BSt1, OffReg} = MMod:get_cp_offset(BSt0), + {BSt2, OffReg2} = MMod:shift_right(BSt1, {free, OffReg}, 2), + % Jump to continuation (this is a tail call) + MMod:jump_to_continuation(BSt2, {free, OffReg2}) + end + ) + end, % Different module: use existing slow path TailCacheKey = {call_primitive_last, ?PRIM_RETURN}, case tail_cache_find(TailCacheKey, TC) of @@ -1241,7 +1265,8 @@ first_pass(<>, MMod, MSt0, State0) -> MSt4 = verify_is_atom(Module, 0, MMod, MSt3), MSt5 = verify_is_atom(Function, 0, MMod, MSt4), MSt6 = MMod:move_to_cp(MSt5, {y_reg, NWords}), - MSt7 = MMod:increment_sp(MSt6, NWords + 1), + % The saved cp occupies one stack slot on 64-bit, two on 32-bit. + MSt7 = MMod:increment_sp(MSt6, NWords + (8 div MMod:word_size())), MSt8 = MMod:call_primitive_last(MSt7, ?PRIM_APPLY, [ ctx, jit_state, offset, {free, Module}, {free, Function}, Arity ]), diff --git a/libs/jit/src/jit_aarch64.erl b/libs/jit/src/jit_aarch64.erl index 02dcdc561d..b8f96d746b 100644 --- a/libs/jit/src/jit_aarch64.erl +++ b/libs/jit/src/jit_aarch64.erl @@ -156,8 +156,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), non_neg_integer()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), regs :: jit_regs:regs() @@ -289,11 +287,9 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, labels = #{}, variant = Variant, - regs = jit_regs:new() + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0) }. %%----------------------------------------------------------------------------- @@ -338,70 +334,14 @@ debugger(#state{stream_module = StreamModule, stream = Stream0} = State) -> Stream1 = StreamModule:append(Stream0, jit_aarch64_asm:brk(0)), State#state{stream = Stream1}. -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently used native registers. This is used for -%% debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of used registers -%%----------------------------------------------------------------------------- --spec used_regs(state()) -> [aarch64_register()]. -used_regs(#state{used_regs = Used}) -> mask_to_list(Used). - -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently available native scratch registers. This -%% is used for debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of available registers -%%----------------------------------------------------------------------------- --spec available_regs(state()) -> [aarch64_register()]. -available_regs(#state{available_regs = Available}) -> mask_to_list(Available). - -%%----------------------------------------------------------------------------- -%% @doc Free native registers. The passed list of registers can contain -%% registers, pointer to registers or other values that are ignored. -%% @end -%% @param State current backend state -%% @param Regs list of registers or other values -%% @return The updated backend state -%%----------------------------------------------------------------------------- --spec free_native_registers(state(), [value()]) -> state(). -free_native_registers(State, []) -> - State; -free_native_registers(State, [Reg | Rest]) -> - State1 = free_native_register(State, Reg), - free_native_registers(State1, Rest). - --spec free_native_register(state(), value()) -> state(). -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Reg -) when - is_atom(Reg) --> - Bit = reg_bit(Reg), - State#state{ - available_regs = Available0 bor Bit, - used_regs = Used0 band (bnot Bit) - }; -free_native_register(State, {ptr, Reg}) -> - free_native_register(State, Reg); -free_native_register(State, _Other) -> - State. - -%%----------------------------------------------------------------------------- -%% @doc Assert that all native scratch registers are available. This is used -%% for debugging and not in production. -%% @end -%% @param State current backend state -%% @return ok -%%----------------------------------------------------------------------------- --spec assert_all_native_free(state()) -> ok. -assert_all_native_free(State) -> - 0 = State#state.used_regs, - ?AVAILABLE_REGS_MASK = State#state.available_regs, - ok. +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1, +%% first_avail/1, mask_to_list/1, args_regs/1, prepare_call_scratch/1) is shared +%% across the register-based backends and flows through jit_regs. +-define(FIRST_AVAIL_REGS, [r7, r8, r9, r10, r11, r12, r13, r14, r15, r3, r4, r5, r6]). +-define(MASK_TO_LIST_REGS, ?FIRST_AVAIL_REGS). +-define(JITSTATE_ARG_REG, ?JITSTATE_REG). +-include("jit_backend_regs_impl.hrl"). %%----------------------------------------------------------------------------- %% @doc Emit the jump table at the beginning of the module. Branches will be @@ -568,15 +508,8 @@ call_primitive_last( % We need a register for the function pointer that should not be used as a parameter % Since we're not returning, we can use all scratch registers except % registers used for parameters - ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), - ArgsRegs = args_regs(Args), - ArgsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), - ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), - ScratchMask = ?AVAILABLE_REGS_MASK band (bnot (ArgsMask bor ParamMask)), - Temp = first_avail(ScratchMask), - TempBit = reg_bit(Temp), - AvailableRegs1 = ScratchMask band (bnot TempBit), - UsedRegs = ?AVAILABLE_REGS_MASK band (bnot AvailableRegs1), + #{temp := Temp, available_mask := AvailableRegs1, used_mask := UsedRegs} = + prepare_call_scratch(Args), PrepCall = case Primitive of 0 -> @@ -588,9 +521,9 @@ call_primitive_last( State1 = set_args( State0#state{ stream = Stream1, - available_regs = AvailableRegs1, - used_regs = UsedRegs, - regs = jit_regs:invalidate_reg(State0#state.regs, Temp) + regs = jit_regs:set_masks( + jit_regs:invalidate_reg(State0#state.regs, Temp), AvailableRegs1, UsedRegs + ) }, Args ), @@ -599,9 +532,9 @@ call_primitive_last( Stream3 = StreamModule:append(Stream2, Call), State1#state{ stream = Stream3, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State1#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State1#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) }. %%----------------------------------------------------------------------------- @@ -618,8 +551,7 @@ return_if_not_equal_to_ctx( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State, {free, Reg} ) -> @@ -635,12 +567,9 @@ return_if_not_equal_to_ctx( I2 = jit_aarch64_asm:bcc(eq, 4 + byte_size(I3) + byte_size(I4)), Stream1 = StreamModule:append(Stream0, <>), Bit = reg_bit(Reg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), State#state{ stream = Stream1, - available_regs = AvailableRegs1, - used_regs = UsedRegs1 + regs = jit_regs:free_reg(Regs0, Bit) }. %%----------------------------------------------------------------------------- @@ -698,10 +627,11 @@ jump_to_continuation( stream_module = StreamModule, stream = Stream0, offset = BaseOffset, - available_regs = Available + regs = Regs0 } = State, {free, OffsetReg} ) -> + Available = jit_regs:available_regs(Regs0), TempReg = first_avail(Available), % Calculate absolute address: native_code_base + target_offset % where native_code_base = current_pc + (BaseOffset - CurrentStreamOffset) @@ -720,9 +650,9 @@ jump_to_continuation( % Free all registers since this is a tail jump State#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(Regs0), ?AVAILABLE_REGS_MASK, 0 + ) }. %% @private @@ -780,9 +710,10 @@ if_block( Stream2, Replacements ), - State4 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State4#state{regs = MergedRegs}; + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}; if_block( #state{stream_module = StreamModule, stream = Stream0} = State0, Cond, @@ -797,9 +728,10 @@ if_block( BranchOffset = OffsetAfter - (Offset + BranchInstrOffset), NewBranchInstr = rewrite_branch_instruction(CC, BranchOffset), Stream3 = StreamModule:replace(Stream2, Offset + BranchInstrOffset, NewBranchInstr), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}. %%----------------------------------------------------------------------------- %% @doc Emit an if else block, i.e. emit a test of a condition and @@ -836,8 +768,6 @@ if_else_block( %% Build the else block StateElse = State2#state{ stream = Stream4, - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs, regs = State1#state.regs }, State3 = BlockFalseFn(StateElse), @@ -847,9 +777,10 @@ if_else_block( FinalJumpOffset = OffsetFinal - ElseJumpOffset, NewElseJumpInstr = jit_aarch64_asm:b(FinalJumpOffset), Stream6 = StreamModule:replace(Stream5, ElseJumpOffset, NewElseJumpInstr), - State4 = merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs), - MergedRegs = jit_regs:merge(State2#state.regs, State3#state.regs), - State4#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State2#state.regs, State3#state.regs, ?AVAILABLE_REGS_MASK + ), + State3#state{stream = Stream6, regs = MergedRegs}. %% @private -spec if_block_cond(state(), condition()) -> @@ -1110,10 +1041,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {RegOrTuple, '&', Val, '!=', 0} ) -> + Available = jit_regs:available_regs(Regs0), Reg = case RegOrTuple of {free, Reg0} -> Reg0; @@ -1144,10 +1076,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {Reg, '&', Mask, '!=', Val} ) when ?IS_GPR(Reg) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), % AND with mask OffsetBefore = StreamModule:offset(Stream0), @@ -1186,23 +1119,15 @@ if_block_cond( %% @private -spec if_block_free_reg(aarch64_register() | {free, aarch64_register()}, state()) -> state(). if_block_free_reg({free, Reg}, State0) -> - #state{available_regs = AvR0, used_regs = UR0} = State0, + #state{regs = Regs0} = State0, Bit = reg_bit(Reg), - AvR1 = AvR0 bor Bit, - UR1 = UR0 band (bnot Bit), State0#state{ - available_regs = AvR1, - used_regs = UR1 + regs = jit_regs:free_reg(Regs0, Bit) }; if_block_free_reg(Reg, State0) when ?IS_GPR(Reg) -> State0. %% @private --spec merge_used_regs(state(), non_neg_integer()) -> state(). -merge_used_regs(#state{used_regs = UR} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - MergedAvail = ?AVAILABLE_REGS_MASK band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. %%----------------------------------------------------------------------------- %% @doc Emit a shift register right by a fixed number of bits, effectively @@ -1227,8 +1152,6 @@ shift_right( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, Reg, @@ -1236,6 +1159,7 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Available = jit_regs:available_regs(Regs0), ResultReg = first_avail(Available), Bit = reg_bit(ResultReg), I = jit_aarch64_asm:lsr(ResultReg, Reg, Shift), @@ -1244,9 +1168,7 @@ shift_right( { State#state{ stream = Stream1, - available_regs = Available band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ResultReg }. @@ -1266,8 +1188,6 @@ shift_right_arith( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, Reg, @@ -1275,6 +1195,7 @@ shift_right_arith( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Available = jit_regs:available_regs(Regs0), ResultReg = first_avail(Available), Bit = reg_bit(ResultReg), I = jit_aarch64_asm:asr(ResultReg, Reg, Shift), @@ -1283,9 +1204,7 @@ shift_right_arith( { State#state{ stream = Stream1, - available_regs = Available band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ResultReg }. @@ -1324,12 +1243,13 @@ call_func_ptr( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State0, FuncPtrTuple, Args ) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), + UsedRegs0 = jit_regs:used_regs(Regs0), FreeRegs = lists:flatmap( fun ({free, ?IP0_REG}) -> []; @@ -1385,14 +1305,12 @@ call_func_ptr( ResultBit = reg_bit(ResultReg), AvailableRegs2 = AvailableRegs1 band (bnot ResultBit), AvailableRegs3 = AvailableRegs2 band ?AVAILABLE_REGS_MASK, - Regs1 = jit_regs:invalidate_all(State0#state.regs), + Regs1 = jit_regs:invalidate_all(Regs0), UsedRegs2 = UsedRegs1 bor ResultBit, { State1#state{ stream = Stream6, - available_regs = AvailableRegs3, - used_regs = UsedRegs2, - regs = Regs1 + regs = jit_regs:set_masks(Regs1, AvailableRegs3, UsedRegs2) }, ResultReg }. @@ -1423,8 +1341,9 @@ pop_registers(false, [RegB, RegA | Tail], StreamModule, Stream0) -> %% @private -spec set_args(state(), [arg()]) -> state(). set_args( - #state{stream = Stream0, stream_module = StreamModule, used_regs = UsedRegs} = State0, Args + #state{stream = Stream0, stream_module = StreamModule, regs = Regs0} = State0, Args ) -> + UsedRegs = jit_regs:used_regs(Regs0), ParamRegs = parameter_regs(Args), ArgsRegs = args_regs(Args), ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), @@ -1453,8 +1372,11 @@ set_args( ), State0#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), - used_regs = ParamMask bor NewUsedMask + regs = jit_regs:set_masks( + Regs0, + ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), + ParamMask bor NewUsedMask + ) }. %% @private @@ -1626,10 +1548,11 @@ move_to_vm_register_emit(State0, Src, {ptr, Reg}) when is_atom(Src) -> Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), State0#state{stream = Stream1}; move_to_vm_register_emit( - #state{available_regs = Available, regs = Regs0} = State0, Src, {y_reg, Y} + #state{regs = Regs0} = State0, Src, {y_reg, Y} ) when is_atom(Src) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), I1 = jit_aarch64_asm:ldr(Temp, ?Y_REGS), I2 = jit_aarch64_asm:str(Src, {Temp, Y * ?WORD_SIZE}), @@ -1639,71 +1562,41 @@ move_to_vm_register_emit( % Source is an integer move_to_vm_register_emit(State, 0, Dest) -> move_to_vm_register_emit(State, xzr, Dest); -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(#state{regs = Regs0} = State0, N, Dest) when is_integer(N) -> - Temp = first_avail(AR0), - TempBit = reg_bit(Temp), - AT = AR0 band (bnot TempBit), - I1 = jit_aarch64_asm:mov(Temp, N), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = AT}, Temp, Dest - ), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {imm, N}), - State1#state{available_regs = AR0, regs = Regs1}; + with_temp(State0, Dest, fun(Temp) -> + {jit_aarch64_asm:mov(Temp, N), jit_regs:set_contents(Regs0, Temp, {imm, N})} + end); % Source is a VM register -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, extra}, Dest) -> - Temp = first_avail(AR0), - TempBit = reg_bit(Temp), - AT = AR0 band (bnot TempBit), - I1 = jit_aarch64_asm:ldr(Temp, ?X_REG(?MAX_REG)), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = AT}, Temp, Dest - ), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, ?MAX_REG}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, X}, Dest) -> - Temp = first_avail(AR0), - TempBit = reg_bit(Temp), - AT = AR0 band (bnot TempBit), - I1 = jit_aarch64_asm:ldr(Temp, ?X_REG(X)), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = AT}, Temp, Dest - ), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, X}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {ptr, Reg}, Dest) -> - Temp = first_avail(AR0), - TempBit = reg_bit(Temp), - AT = AR0 band (bnot TempBit), - I1 = jit_aarch64_asm:ldr(Temp, {Reg, 0}), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = AT}, Temp, Dest - ), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {y_reg, Y}, Dest) -> - Temp = first_avail(AR0), - TempBit = reg_bit(Temp), - AT = AR0 band (bnot TempBit), - I1 = jit_aarch64_asm:ldr(Temp, ?Y_REGS), - I2 = jit_aarch64_asm:ldr(Temp, {Temp, Y * ?WORD_SIZE}), - Stream1 = (State0#state.stream_module):append(State0#state.stream, <>), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = AT}, Temp, Dest - ), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {y_reg, Y}), - State1#state{available_regs = AR0, regs = Regs1}; +move_to_vm_register_emit(#state{regs = Regs0} = State0, {x_reg, extra}, Dest) -> + with_temp(State0, Dest, fun(Temp) -> + { + jit_aarch64_asm:ldr(Temp, ?X_REG(?MAX_REG)), + jit_regs:set_contents(Regs0, Temp, {x_reg, ?MAX_REG}) + } + end); +move_to_vm_register_emit(#state{regs = Regs0} = State0, {x_reg, X}, Dest) -> + with_temp(State0, Dest, fun(Temp) -> + {jit_aarch64_asm:ldr(Temp, ?X_REG(X)), jit_regs:set_contents(Regs0, Temp, {x_reg, X})} + end); +move_to_vm_register_emit(#state{regs = Regs0} = State0, {ptr, Reg}, Dest) -> + with_temp(State0, Dest, fun(Temp) -> + {jit_aarch64_asm:ldr(Temp, {Reg, 0}), jit_regs:invalidate_reg(Regs0, Temp)} + end); +move_to_vm_register_emit(#state{regs = Regs0} = State0, {y_reg, Y}, Dest) -> + with_temp(State0, Dest, fun(Temp) -> + I1 = jit_aarch64_asm:ldr(Temp, ?Y_REGS), + I2 = jit_aarch64_asm:ldr(Temp, {Temp, Y * ?WORD_SIZE}), + {<>, jit_regs:set_contents(Regs0, Temp, {y_reg, Y})} + end); % term_to_float move_to_vm_register_emit( - #state{stream_module = StreamModule, available_regs = Available, stream = Stream0} = State0, + #state{stream_module = StreamModule, regs = Regs0, stream = Stream0} = State0, {free, {ptr, Reg, 1}}, {fp_reg, F} ) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), I1 = jit_aarch64_asm:ldr(Reg, {Reg, ?WORD_SIZE}), I2 = jit_aarch64_asm:ldr(Temp, ?FP_REGS), @@ -1714,6 +1607,29 @@ move_to_vm_register_emit( Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), State1#state{stream = Stream1, regs = Regs1}. +-spec with_temp( + state(), + vm_register(), + fun((aarch64_register()) -> {binary(), jit_regs:regs()}) +) -> state(). +with_temp( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Dest, EmitFn +) -> + AR0 = jit_regs:available_regs(Regs0), + Temp = first_avail(AR0), + TempBit = reg_bit(Temp), + {Code, Regs1} = EmitFn(Temp), + Stream1 = StreamModule:append(Stream0, Code), + State1 = move_to_vm_register_emit( + State0#state{ + stream = Stream1, + regs = jit_regs:set_available_regs(Regs1, AR0 band (bnot TempBit)) + }, + Temp, + Dest + ), + State1#state{regs = jit_regs:set_available_regs(State1#state.regs, AR0)}. + %%----------------------------------------------------------------------------- %% @doc Emit a move of an array element (reg[x]) to a vm or a native register. %% @end @@ -1730,12 +1646,13 @@ move_to_vm_register_emit( vm_register() | aarch64_register() ) -> state(). move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), I1 = jit_aarch64_asm:ldr(Temp, {Reg, Index * ?WORD_SIZE}), I2 = jit_aarch64_asm:str(Temp, ?X_REG(X)), @@ -1744,12 +1661,13 @@ move_array_element( Regs2 = jit_regs:set_contents(Regs1, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), I1 = jit_aarch64_asm:ldr(Temp, {Reg, Index * ?WORD_SIZE}), I2 = jit_aarch64_asm:str(Temp, {Dest, 0}), @@ -1757,12 +1675,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) -> + Available = jit_regs:available_regs(Regs0), Temp1 = first_avail(Available), Bit1 = reg_bit(Temp1), Avail1 = Available band (bnot Bit1), @@ -1777,12 +1696,13 @@ move_array_element( Regs3 = jit_regs:invalidate_reg(Regs2, Temp2), State#state{stream = Stream1, regs = Regs3}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Index, {y_reg, Y} ) when is_integer(Index) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), I1 = jit_aarch64_asm:ldr(Temp, ?Y_REGS), I2 = jit_aarch64_asm:ldr(Reg, {Reg, Index * ?WORD_SIZE}), @@ -1804,8 +1724,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -1815,23 +1733,17 @@ move_array_element( I1 = jit_aarch64_asm:ldr(IndexReg, {Reg, IndexReg, lsl, 3}), I2 = jit_aarch64_asm:str(IndexReg, ?X_REG(X)), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_vm_loc(Regs0, {x_reg, X}), Regs2 = jit_regs:invalidate_reg(Regs1, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs2 + regs = jit_regs:free_reg(Regs2, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -1841,35 +1753,28 @@ move_array_element( I1 = jit_aarch64_asm:ldr(IndexReg, {Reg, IndexReg, lsl, 3}), I2 = jit_aarch64_asm:str(IndexReg, {PtrReg, 0}), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_reg(Regs0, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs1 + regs = jit_regs:free_reg(Regs1, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, {free, IndexReg}, {y_reg, Y} ) when ?IS_GPR(IndexReg) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), Temp = first_avail(AvailableRegs0), I1 = jit_aarch64_asm:ldr(Temp, ?Y_REGS), I2 = jit_aarch64_asm:ldr(IndexReg, {Reg, IndexReg, lsl, 3}), I3 = jit_aarch64_asm:str(IndexReg, {Temp, Y * ?WORD_SIZE}), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append( Stream0, <> ), @@ -1877,10 +1782,8 @@ move_array_element( Regs2 = jit_regs:invalidate_reg(Regs1, Temp), Regs3 = jit_regs:invalidate_reg(Regs2, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs3 + regs = jit_regs:free_reg(Regs3, Bit) }. %%----------------------------------------------------------------------------- @@ -1912,13 +1815,12 @@ get_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, Index ) -> + Available = jit_regs:available_regs(Regs0), ElemReg = first_avail(Available), Bit = reg_bit(ElemReg), I1 = jit_aarch64_asm:ldr(ElemReg, {Reg, Index * ?WORD_SIZE}), @@ -1927,9 +1829,7 @@ get_array_element( { State#state{ stream = Stream1, - available_regs = Available band (bnot Bit), - used_regs = UsedRegs0 bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ElemReg }. @@ -1998,13 +1898,14 @@ move_to_array_element( ) when is_integer(IndexVal) andalso is_integer(Offset) -> move_to_array_element(State, Value, BaseReg, IndexVal + Offset); move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, ValueReg, BaseReg, IndexReg, Offset ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), I1 = jit_aarch64_asm:add(Temp, IndexReg, Offset), I2 = jit_aarch64_asm:str(ValueReg, {BaseReg, Temp, lsl, 3}), @@ -2019,7 +1920,7 @@ move_to_array_element( Offset ) -> {State1, ValueReg} = copy_to_native_register(State0, Value), - Temp = first_avail(State1#state.available_regs), + Temp = first_avail(jit_regs:available_regs(State1#state.regs)), I1 = jit_aarch64_asm:add(Temp, IndexReg, Offset), I2 = jit_aarch64_asm:str(ValueReg, {BaseReg, Temp, lsl, 3}), Stream1 = (State1#state.stream_module):append(State1#state.stream, <>), @@ -2044,16 +1945,17 @@ move_to_native_register(#state{regs = Regs} = State, Value) -> case Contents =/= unknown andalso jit_regs:find_reg_with_contents(Regs, Contents) of {ok, CachedReg} -> Bit = reg_bit(CachedReg), - case State#state.used_regs band Bit of + CurUsed = jit_regs:used_regs(Regs), + CurAvail = jit_regs:available_regs(Regs), + case CurUsed band Bit of 0 -> - case State#state.available_regs band Bit of + case CurAvail band Bit of 0 -> move_to_native_register_emit(State, Value, Contents); _ -> { State#state{ - used_regs = State#state.used_regs bor Bit, - available_regs = State#state.available_regs band (bnot Bit) + regs = jit_regs:alloc_reg(Regs, Bit) }, CachedReg } @@ -2069,13 +1971,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, cp, Contents ) -> + Available = jit_regs:available_regs(Regs0), Reg = first_avail(Available), Bit = reg_bit(Reg), I1 = jit_aarch64_asm:ldr(Reg, ?CP), @@ -2084,9 +1985,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Available band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2103,8 +2002,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, Imm, @@ -2112,6 +2009,7 @@ move_to_native_register_emit( ) when is_integer(Imm) -> + Available = jit_regs:available_regs(Regs0), Reg = first_avail(Available), Bit = reg_bit(Reg), I1 = jit_aarch64_asm:mov(Reg, Imm), @@ -2120,9 +2018,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Available band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2130,13 +2026,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, {x_reg, extra}, Contents ) -> + Available = jit_regs:available_regs(Regs0), Reg = first_avail(Available), Bit = reg_bit(Reg), I1 = jit_aarch64_asm:ldr(Reg, ?X_REG(?MAX_REG)), @@ -2145,9 +2040,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Available band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2155,8 +2048,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, {x_reg, X}, @@ -2164,6 +2055,7 @@ move_to_native_register_emit( ) when X < ?MAX_REG -> + Available = jit_regs:available_regs(Regs0), Reg = first_avail(Available), Bit = reg_bit(Reg), I1 = jit_aarch64_asm:ldr(Reg, ?X_REG(X)), @@ -2172,9 +2064,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Available band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2182,13 +2072,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, {y_reg, Y}, Contents ) -> + Available = jit_regs:available_regs(Regs0), Reg = first_avail(Available), Bit = reg_bit(Reg), I1 = jit_aarch64_asm:ldr(Reg, ?Y_REGS), @@ -2199,9 +2088,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - available_regs = Available band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }. @@ -2280,12 +2167,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, Reg ) when is_atom(Reg) -> + Available = jit_regs:available_regs(Regs0), SaveReg = first_avail(Available), Bit = reg_bit(SaveReg), I1 = jit_aarch64_asm:mov(SaveReg, Reg), @@ -2295,9 +2181,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Available band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, SaveReg }; @@ -2305,12 +2189,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used, regs = Regs0 } = State, {ptr, Reg} ) when is_atom(Reg) -> + Available = jit_regs:available_regs(Regs0), SaveReg = first_avail(Available), Bit = reg_bit(SaveReg), I1 = jit_aarch64_asm:ldr(SaveReg, {Reg, 0}), @@ -2319,9 +2202,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Available band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, SaveReg }; @@ -2337,10 +2218,10 @@ copy_to_native_register(State, Reg) -> %%----------------------------------------------------------------------------- -spec move_to_cp(state(), vm_register()) -> state(). move_to_cp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y} ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), I1 = jit_aarch64_asm:ldr(Reg, ?Y_REGS), I2 = jit_aarch64_asm:ldr(Reg, {Reg, Y * ?WORD_SIZE}), @@ -2359,10 +2240,10 @@ move_to_cp( %%----------------------------------------------------------------------------- -spec increment_sp(state(), integer()) -> state(). increment_sp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Offset ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), I1 = jit_aarch64_asm:ldr(Reg, ?Y_REGS), I2 = jit_aarch64_asm:add(Reg, Reg, Offset * ?WORD_SIZE), @@ -2385,13 +2266,13 @@ set_continuation_to_label( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, labels = Labels, regs = Regs0 } = State, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Offset = StreamModule:offset(Stream0), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), @@ -2430,11 +2311,11 @@ set_continuation_to_offset( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), OffsetRef = make_ref(), Offset = StreamModule:offset(Stream0), @@ -2476,11 +2357,10 @@ get_module_index( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_aarch64_asm:ldr(Reg, ?JITSTATE_MODULE), @@ -2491,9 +2371,7 @@ get_module_index( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UsedRegs0 bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }. @@ -2507,7 +2385,7 @@ op_imm(#state{stream_module = StreamModule, stream = Stream0} = State, Op, Reg, StreamModule:append(Stream0, I) catch error:{unencodable_immediate, Val} -> - Temp = first_avail(State#state.available_regs), + Temp = first_avail(jit_regs:available_regs(State#state.regs)), I1 = jit_aarch64_asm:mov(Temp, Val), I2 = jit_aarch64_asm:Op(Reg, Reg, Temp), StreamModule:append(Stream0, <>) @@ -2550,21 +2428,29 @@ and_(#state{regs = Regs0} = State, {free, Reg}, Val) -> Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {NewState#state{regs = Regs1}, Reg}; and_( - #state{available_regs = Avail, used_regs = UR, regs = Regs0} = State, + #state{regs = Regs0} = State, Reg, Val ) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), NewState = op_imm( - State#state{available_regs = Avail band (bnot Bit), used_regs = UR bor Bit}, + State#state{ + regs = jit_regs:alloc_reg(Regs0, Bit) + }, and_, ResultReg, Reg, Val ), Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), - {NewState#state{regs = Regs1}, ResultReg}. + NewRegs = jit_regs:set_masks( + Regs1, + jit_regs:available_regs(NewState#state.regs), + jit_regs:used_regs(NewState#state.regs) + ), + {NewState#state{regs = NewRegs}, ResultReg}. %%----------------------------------------------------------------------------- %% @doc Perform bitwise OR of a register with an immediate value. @@ -2647,7 +2533,8 @@ mul(State, _Reg, 1) -> State; mul(State, Reg, 2) -> shift_left(State, Reg, 1); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> +mul(#state{regs = Regs0} = State, Reg, 3) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_aarch64_asm:lsl(Temp, Reg, 1), I2 = jit_aarch64_asm:add(Reg, Temp, Reg), @@ -2656,7 +2543,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 4) -> shift_left(State, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> +mul(#state{regs = Regs0} = State, Reg, 5) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_aarch64_asm:lsl(Temp, Reg, 2), I2 = jit_aarch64_asm:add(Reg, Temp, Reg), @@ -2666,7 +2554,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> mul(State0, Reg, 6) -> State1 = mul(State0, Reg, 3), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> +mul(#state{regs = Regs0} = State, Reg, 7) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_aarch64_asm:lsl(Temp, Reg, 3), I2 = jit_aarch64_asm:sub(Reg, Temp, Reg), @@ -2675,7 +2564,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 8) -> shift_left(State, Reg, 3); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> +mul(#state{regs = Regs0} = State, Reg, 9) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_aarch64_asm:lsl(Temp, Reg, 3), I2 = jit_aarch64_asm:add(Reg, Temp, Reg), @@ -2685,7 +2575,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> mul(State0, Reg, 10) -> State1 = mul(State0, Reg, 5), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 15) -> +mul(#state{regs = Regs0} = State, Reg, 15) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_aarch64_asm:lsl(Temp, Reg, 4), I2 = jit_aarch64_asm:sub(Reg, Temp, Reg), @@ -2699,11 +2590,12 @@ mul(State, Reg, 32) -> mul(State, Reg, 64) -> shift_left(State, Reg, 6); mul( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Val ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_aarch64_asm:mov(Temp, Val), I2 = jit_aarch64_asm:mul(Reg, Reg, Temp), @@ -2731,11 +2623,12 @@ div_( -spec rem_(state(), aarch64_register(), aarch64_register()) -> {state(), aarch64_register()}. rem_( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DividendReg, DivisorReg ) -> + Avail = jit_regs:available_regs(Regs0), %% rem = dividend - (dividend / divisor) * divisor %% Use msub: Rd = Ra - (Rn * Rm) %% First sdiv into a temp, then msub @@ -2756,9 +2649,10 @@ rem_( %%----------------------------------------------------------------------------- -spec decrement_reductions_and_maybe_schedule_next(state()) -> state(). decrement_reductions_and_maybe_schedule_next( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0 ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), % Load reduction count @@ -2787,9 +2681,8 @@ decrement_reductions_and_maybe_schedule_next( Stream4 = StreamModule:replace( Stream3, BNEOffset, <> ), - State3 = merge_used_regs(State2#state{stream = Stream4}, State1#state.used_regs), %% schedule_next clobbers caller-saved regs; invalidate cache at continuation. - State3#state{regs = jit_regs:invalidate_all(State1#state.regs)}. + State2#state{stream = Stream4, regs = jit_regs:invalidate_all(State1#state.regs)}. %%----------------------------------------------------------------------------- %% @doc Emit a call to a label with automatic scheduling. Decrements reductions @@ -2822,10 +2715,11 @@ call_only_or_schedule_next( stream = Stream0, branches = Branches, labels = Labels, - available_regs = Avail + regs = Regs0 } = State0, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % Load reduction count I1 = jit_aarch64_asm:ldr_w(Temp, ?JITSTATE_REDUCTIONCOUNT), @@ -2984,78 +2878,6 @@ reg_bit(r15) -> ?REG_BIT_R15; reg_bit(r16) -> ?REG_BIT_R16; reg_bit(r17) -> ?REG_BIT_R17. -%% first_avail returns the first available register from a bitmask. -%% Order: [r7, r8, r9, r10, r11, r12, r13, r14, r15, r3, r4, r5, r6] -first_avail(Mask) when Mask band ?REG_BIT_R7 =/= 0 -> r7; -first_avail(Mask) when Mask band ?REG_BIT_R8 =/= 0 -> r8; -first_avail(Mask) when Mask band ?REG_BIT_R9 =/= 0 -> r9; -first_avail(Mask) when Mask band ?REG_BIT_R10 =/= 0 -> r10; -first_avail(Mask) when Mask band ?REG_BIT_R11 =/= 0 -> r11; -first_avail(Mask) when Mask band ?REG_BIT_R12 =/= 0 -> r12; -first_avail(Mask) when Mask band ?REG_BIT_R13 =/= 0 -> r13; -first_avail(Mask) when Mask band ?REG_BIT_R14 =/= 0 -> r14; -first_avail(Mask) when Mask band ?REG_BIT_R15 =/= 0 -> r15; -first_avail(Mask) when Mask band ?REG_BIT_R3 =/= 0 -> r3; -first_avail(Mask) when Mask band ?REG_BIT_R4 =/= 0 -> r4; -first_avail(Mask) when Mask band ?REG_BIT_R5 =/= 0 -> r5; -first_avail(Mask) when Mask band ?REG_BIT_R6 =/= 0 -> r6. - -%% Convert bitmask to list, matching the order -mask_to_list(0) -> []; -mask_to_list(Mask) -> mask_to_list_r7(Mask). - -mask_to_list_r7(Mask) when Mask band ?REG_BIT_R7 =/= 0 -> [r7 | mask_to_list_r8(Mask)]; -mask_to_list_r7(Mask) -> mask_to_list_r8(Mask). -mask_to_list_r8(Mask) when Mask band ?REG_BIT_R8 =/= 0 -> [r8 | mask_to_list_r9(Mask)]; -mask_to_list_r8(Mask) -> mask_to_list_r9(Mask). -mask_to_list_r9(Mask) when Mask band ?REG_BIT_R9 =/= 0 -> [r9 | mask_to_list_r10(Mask)]; -mask_to_list_r9(Mask) -> mask_to_list_r10(Mask). -mask_to_list_r10(Mask) when Mask band ?REG_BIT_R10 =/= 0 -> [r10 | mask_to_list_r11(Mask)]; -mask_to_list_r10(Mask) -> mask_to_list_r11(Mask). -mask_to_list_r11(Mask) when Mask band ?REG_BIT_R11 =/= 0 -> [r11 | mask_to_list_r12(Mask)]; -mask_to_list_r11(Mask) -> mask_to_list_r12(Mask). -mask_to_list_r12(Mask) when Mask band ?REG_BIT_R12 =/= 0 -> [r12 | mask_to_list_r13(Mask)]; -mask_to_list_r12(Mask) -> mask_to_list_r13(Mask). -mask_to_list_r13(Mask) when Mask band ?REG_BIT_R13 =/= 0 -> [r13 | mask_to_list_r14(Mask)]; -mask_to_list_r13(Mask) -> mask_to_list_r14(Mask). -mask_to_list_r14(Mask) when Mask band ?REG_BIT_R14 =/= 0 -> [r14 | mask_to_list_r15(Mask)]; -mask_to_list_r14(Mask) -> mask_to_list_r15(Mask). -mask_to_list_r15(Mask) when Mask band ?REG_BIT_R15 =/= 0 -> [r15 | mask_to_list_r3(Mask)]; -mask_to_list_r15(Mask) -> mask_to_list_r3(Mask). -mask_to_list_r3(Mask) when Mask band ?REG_BIT_R3 =/= 0 -> [r3 | mask_to_list_r4(Mask)]; -mask_to_list_r3(Mask) -> mask_to_list_r4(Mask). -mask_to_list_r4(Mask) when Mask band ?REG_BIT_R4 =/= 0 -> [r4 | mask_to_list_r5(Mask)]; -mask_to_list_r4(Mask) -> mask_to_list_r5(Mask). -mask_to_list_r5(Mask) when Mask band ?REG_BIT_R5 =/= 0 -> [r5 | mask_to_list_r6(Mask)]; -mask_to_list_r5(Mask) -> mask_to_list_r6(Mask). -mask_to_list_r6(Mask) when Mask band ?REG_BIT_R6 =/= 0 -> [r6]; -mask_to_list_r6(_Mask) -> []. - -%% @private --spec args_regs([arg()]) -> [aarch64_register() | imm]. -args_regs(Args) -> - lists:map( - fun - ({free, {ptr, Reg}}) -> Reg; - ({free, Reg}) when is_atom(Reg) -> Reg; - ({free, Imm}) when is_integer(Imm) -> imm; - (offset) -> imm; - (ctx) -> ?CTX_REG; - (jit_state) -> ?JITSTATE_REG; - (Reg) when is_atom(Reg) -> Reg; - (Imm) when is_integer(Imm) -> imm; - ({ptr, Reg}) -> Reg; - ({x_reg, _}) -> ?CTX_REG; - ({y_reg, _}) -> ?CTX_REG; - ({fp_reg, _}) -> ?CTX_REG; - ({free, {x_reg, _}}) -> ?CTX_REG; - ({free, {y_reg, _}}) -> ?CTX_REG; - ({free, {fp_reg, _}}) -> ?CTX_REG; - ({avm_int64_t, _}) -> imm - end, - Args - ). - %%----------------------------------------------------------------------------- %% @doc Add a label at the current offset %% @end diff --git a/libs/jit/src/jit_arm32.erl b/libs/jit/src/jit_arm32.erl index 281d7fb276..877b52ceb3 100644 --- a/libs/jit/src/jit_arm32.erl +++ b/libs/jit/src/jit_arm32.erl @@ -60,6 +60,9 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -143,8 +146,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), non_neg_integer()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), literal_pool :: [{non_neg_integer(), arm32_register(), non_neg_integer()}], @@ -181,10 +182,13 @@ -define(NATIVE_INTERFACE_REG, r2). -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). +% ctx->cp is a 64-bit cp_t occupying two slots (little-endian targets): +% ?CP holds the low word (offset << 2), ?CP_MODULE holds the high word (Module*). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). % JITSTATE is on stack, accessed via stack offset % These macros now expect a register that contains the jit_state pointer -define(JITSTATE_MODULE(Reg), {Reg, 0}). @@ -292,12 +296,10 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, labels = #{}, variant = Variant, literal_pool = [], - regs = jit_regs:new() + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0) }. %%----------------------------------------------------------------------------- @@ -342,69 +344,14 @@ debugger(#state{stream_module = StreamModule, stream = Stream0} = State) -> Stream1 = StreamModule:append(Stream0, jit_arm32_asm:bkpt(0)), State#state{stream = Stream1}. -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently used native registers. This is used for -%% debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of used registers -%%----------------------------------------------------------------------------- --spec used_regs(state()) -> [arm32_register()]. -used_regs(#state{used_regs = Used}) -> mask_to_list(Used). - -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently available native scratch registers. This -%% is used for debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of available registers -%%----------------------------------------------------------------------------- --spec available_regs(state()) -> [arm32_register()]. -available_regs(#state{available_regs = Available}) -> mask_to_list(Available). - -%%----------------------------------------------------------------------------- -%% @doc Free native registers. The passed list of registers can contain -%% registers, pointer to registers or other values that are ignored. -%% @end -%% @param State current backend state -%% @param Regs list of registers or other values -%% @return The updated backend state -%%----------------------------------------------------------------------------- --spec free_native_registers(state(), [value()]) -> state(). -free_native_registers(State, []) -> - State; -free_native_registers(State, [Reg | Rest]) -> - State1 = free_native_register(State, Reg), - free_native_registers(State1, Rest). - --spec free_native_register(state(), value()) -> state(). -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Reg -) when - is_atom(Reg) --> - Bit = reg_bit(Reg), - State#state{ - available_regs = Available0 bor Bit, used_regs = Used0 band (bnot Bit) - }; -free_native_register(State, {ptr, Reg}) -> - free_native_register(State, Reg); -free_native_register(State, _Other) -> - State. - -%%----------------------------------------------------------------------------- -%% @doc Assert that all native scratch registers are available. This is used -%% for debugging and not in production. -%% @end -%% @param State current backend state -%% @return ok -%%----------------------------------------------------------------------------- --spec assert_all_native_free(state()) -> ok. -assert_all_native_free(State) -> - 0 = State#state.used_regs, - ?AVAILABLE_REGS_MASK = State#state.available_regs, - ok. +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1, +%% first_avail/1, mask_to_list/1, args_regs/1, prepare_call_scratch/1) is shared +%% across the register-based backends and flows through jit_regs. +-define(FIRST_AVAIL_REGS, ?AVAILABLE_REGS). +-define(MASK_TO_LIST_REGS, ?FIRST_AVAIL_REGS). +-define(JITSTATE_ARG_REG, jit_state). +-include("jit_backend_regs_impl.hrl"). %%----------------------------------------------------------------------------- %% @doc Emit the jump table at the beginning of the module. Branches will be @@ -586,31 +533,28 @@ call_primitive( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used - } = State, - Primitive, - Args -) when Available =/= 0 -> - % Use an available register for loading the function pointer - TempReg = first_avail(Available), - TempBit = reg_bit(TempReg), - PrepCall = load_primitive_ptr(Primitive, TempReg), - Stream1 = StreamModule:append(Stream0, PrepCall), - Regs0 = jit_regs:invalidate_reg(State#state.regs, TempReg), - StateCall = State#state{ - stream = Stream1, - available_regs = Available band (bnot TempBit), - used_regs = Used bor TempBit, regs = Regs0 - }, - call_func_ptr(StateCall, {free, TempReg}, Args); -call_primitive( - #state{available_regs = 0} = State, + } = State, Primitive, Args ) -> - call_func_ptr(State, {primitive, Primitive}, Args). + Available = jit_regs:available_regs(Regs0), + case Available of + 0 -> + call_func_ptr(State, {primitive, Primitive}, Args); + _ -> + % Use an available register for loading the function pointer + TempReg = first_avail(Available), + TempBit = reg_bit(TempReg), + PrepCall = load_primitive_ptr(Primitive, TempReg), + Stream1 = StreamModule:append(Stream0, PrepCall), + Regs1 = jit_regs:invalidate_reg(Regs0, TempReg), + StateCall = State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, TempBit) + }, + call_func_ptr(StateCall, {free, TempReg}, Args) + end. %%----------------------------------------------------------------------------- %% @doc Emit a jump (call without return) to a primitive with arguments. This @@ -634,15 +578,8 @@ call_primitive_last( % We need a register for the function pointer that should not be used as a parameter % Since we're not returning, we can use all scratch registers except % registers used for parameters - ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), - ArgsRegs = args_regs(Args), - ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), - ArgsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), - ScratchMask = ?AVAILABLE_REGS_MASK band (bnot (ArgsMask bor ParamMask)), - Temp = first_avail(ScratchMask), - TempBit = reg_bit(Temp), - AvailableRegs1 = ScratchMask band (bnot TempBit), - UsedRegs = ?AVAILABLE_REGS_MASK band (bnot AvailableRegs1), + #{temp := Temp, available_mask := AvailableRegs1, used_mask := UsedRegs} = + prepare_call_scratch(Args), PrepCall = load_primitive_ptr(Primitive, Temp), Stream1 = StreamModule:append(Stream0, PrepCall), @@ -650,7 +587,8 @@ call_primitive_last( % not whatever value the cache may have recorded for it. Regs1 = jit_regs:invalidate_reg(State0#state.regs, Temp), State1 = State0#state{ - stream = Stream1, available_regs = AvailableRegs1, used_regs = UsedRegs, regs = Regs1 + stream = Stream1, + regs = jit_regs:set_masks(Regs1, AvailableRegs1, UsedRegs) }, % Preprocess offset special arg @@ -694,9 +632,9 @@ call_primitive_last( tail_call_with_jit_state_registers_only(State2, Temp) end, State5 = State4#state{ - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State4#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State4#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) }, flush_literal_pool(State5). @@ -750,8 +688,6 @@ return_if_not_equal_to_ctx( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, {free, Reg} @@ -770,9 +706,7 @@ return_if_not_equal_to_ctx( RegBit = reg_bit(Reg), State#state{ stream = Stream1, - available_regs = AvailableRegs0 bor RegBit, - used_regs = UsedRegs0 band (bnot RegBit), - regs = Regs0 + regs = jit_regs:free_reg(Regs0, RegBit) }. %%----------------------------------------------------------------------------- @@ -820,11 +754,12 @@ jump_to_continuation( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, + regs = Regs0, offset = BaseOffset } = State0, {free, OffsetReg} ) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), % ARM32 PIC implementation: % 1. Use mov(al, Temp, pc) to read PC (gives current instruction address + 8) @@ -867,9 +802,9 @@ jump_to_continuation( % Free all registers as this is a terminal instruction State2 = State1#state{ stream = Stream2, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State1#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State1#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) }, flush_literal_pool(State2). @@ -926,9 +861,10 @@ if_block( Stream2, Replacements ), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}; + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}; if_block( #state{stream_module = StreamModule, stream = Stream0} = State0, Cond, @@ -943,9 +879,10 @@ if_block( BranchOffset = OffsetAfter - (Offset + BranchInstrOffset), NewBranchInstr = jit_arm32_asm:b(CC, BranchOffset), Stream3 = StreamModule:replace(Stream2, Offset + BranchInstrOffset, NewBranchInstr), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}. %%----------------------------------------------------------------------------- %% @doc Emit an if else block, i.e. emit a test of a condition and @@ -983,8 +920,6 @@ if_else_block( %% Build the else block StateElse = State2#state{ stream = Stream4, - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs, regs = State1#state.regs }, State3 = BlockFalseFn(StateElse), @@ -994,9 +929,10 @@ if_else_block( FinalJumpOffset = OffsetFinal - ElseJumpOffset, NewElseJumpInstr = jit_arm32_asm:b(al, FinalJumpOffset), Stream6 = StreamModule:replace(Stream5, ElseJumpOffset, NewElseJumpInstr), - State4 = merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs), - MergedRegs = jit_regs:merge(State2#state.regs, State3#state.regs), - State4#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State2#state.regs, State3#state.regs, ?AVAILABLE_REGS_MASK + ), + State3#state{stream = Stream6, regs = MergedRegs}. -spec if_block_cond(state(), condition()) -> {state(), jit_arm32_asm:cc(), non_neg_integer()}. @@ -1021,9 +957,10 @@ if_block_cond( {State2, le, byte_size(I1)}; %% Handle {Val, '<', Reg} for values > 255, need to load into temp register if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {Val, '<', RegOrTuple} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1078,9 +1015,10 @@ if_block_cond( State2 = State1#state{stream = Stream1}, {State2, CC, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '<', Val} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1101,9 +1039,10 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, CC, Offset1 - Offset0 + byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, available_regs = Available} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {Val, '<', RegOrTuple} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1208,9 +1147,10 @@ if_block_cond( State3 = if_block_free_reg({free, RegB}, State2), {State3, CC, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '==', Val} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Offset0 = StreamModule:offset(Stream0), Reg = @@ -1230,9 +1170,10 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, CC, Offset1 - Offset0 + byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '!=', Val} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Offset0 = StreamModule:offset(Stream0), Reg = @@ -1254,8 +1195,7 @@ if_block_cond( if_block_cond( #state{ stream_module = StreamModule, - stream = Stream0, - available_regs = _Available + stream = Stream0 } = State0, {'(bool)', RegOrTuple, '==', false} ) -> @@ -1298,10 +1238,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {RegOrTuple, '&', Val, '!=', 0} -) when Available =/= 0 -> +) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1332,10 +1273,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {Reg, '&', 16#F, '!=', 16#F} -) when ?IS_GPR(Reg), Available =/= 0 -> +) when ?IS_GPR(Reg) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), % Special case Reg & ?TERM_IMMED_TAG_MASK != ?TERM_INTEGER_TAG I1 = jit_arm32_asm:and_(al, Temp, Reg, 16#F), @@ -1367,10 +1309,12 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {Reg, '&', Mask, '!=', Val} -) when ?IS_GPR(Reg), Available =/= 0 -> +) when ?IS_GPR(Reg) -> + Available = jit_regs:available_regs(Regs0), + Used = jit_regs:used_regs(Regs0), Temp = first_avail(Available), TempBit = reg_bit(Temp), AT = Available band (bnot TempBit), @@ -1379,7 +1323,9 @@ if_block_cond( I1 = jit_arm32_asm:mov(al, Temp, Reg), Stream1 = StreamModule:append(Stream0, I1), State1 = State0#state{stream = Stream1}, - {State2, Temp} = and_(State1#state{available_regs = AT}, {free, Temp}, Mask), + {State2, Temp} = and_( + State1#state{regs = jit_regs:set_masks(Regs0, AT, Used)}, {free, Temp}, Mask + ), Stream2 = State2#state.stream, % Compare with value I2 = jit_arm32_asm:cmp(al, Temp, Val), @@ -1390,7 +1336,10 @@ if_block_cond( Stream4 = StreamModule:append(Stream3, <<16#FFFFFFFF:32>>), Regs3 = jit_regs:invalidate_reg(State2#state.regs, Temp), State3 = State2#state{ - stream = Stream4, available_regs = State2#state.available_regs bor TempBit, regs = Regs3 + stream = Stream4, + regs = jit_regs:set_available_regs( + Regs3, jit_regs:available_regs(State2#state.regs) bor TempBit + ) }, {State3, CC, OffsetAfter - OffsetBefore}; if_block_cond( @@ -1417,23 +1366,14 @@ if_block_cond( -spec if_block_free_reg(arm32_register() | {free, arm32_register()}, state()) -> state(). if_block_free_reg({free, Reg}, State0) -> - #state{available_regs = AvR0, used_regs = UR0} = State0, + #state{regs = Regs0} = State0, Bit = reg_bit(Reg), - AvR1 = AvR0 bor Bit, - UR1 = UR0 band (bnot Bit), State0#state{ - available_regs = AvR1, - used_regs = UR1 + regs = jit_regs:free_reg(Regs0, Bit) }; if_block_free_reg(Reg, State0) when ?IS_GPR(Reg) -> State0. --spec merge_used_regs(state(), non_neg_integer()) -> state(). -merge_used_regs(#state{used_regs = UR} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - MergedAvail = ?AVAILABLE_REGS_MASK band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. - %%----------------------------------------------------------------------------- %% @doc Emit a shift register right by a fixed number of bits, effectively %% dividing it by 2^Shift @@ -1457,8 +1397,6 @@ shift_right( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1466,6 +1404,7 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I = jit_arm32_asm:lsr(al, ResultReg, Reg, Shift), @@ -1474,9 +1413,7 @@ shift_right( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ResultReg }. @@ -1496,8 +1433,6 @@ shift_right_arith( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1505,6 +1440,7 @@ shift_right_arith( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I = jit_arm32_asm:asr(al, ResultReg, Reg, Shift), @@ -1513,9 +1449,7 @@ shift_right_arith( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ResultReg }. @@ -1553,12 +1487,13 @@ call_func_ptr( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0Mask, - used_regs = UsedRegs0Mask + regs = Regs0 } = State0, FuncPtrTuple, Args ) -> + AvailableRegs0Mask = jit_regs:available_regs(Regs0), + UsedRegs0Mask = jit_regs:used_regs(Regs0), AvailableRegs0 = mask_to_list(AvailableRegs0Mask), UsedRegs0 = mask_to_list(UsedRegs0Mask), FreeRegs = lists:flatmap( @@ -1612,9 +1547,12 @@ call_func_ptr( % and the currently available registers to push values to the stack. SetArgsPushStackAvailableArgs = (UsedRegs1 -- (RegArgsRegs ++ StackArgsRegs)) ++ AvailableRegs0, State1 = State0#state{ - available_regs = jit_regs:regs_to_mask(SetArgsPushStackAvailableArgs, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask( - ?AVAILABLE_REGS -- SetArgsPushStackAvailableArgs, fun reg_bit/1 + regs = jit_regs:set_masks( + Regs0, + jit_regs:regs_to_mask(SetArgsPushStackAvailableArgs, fun reg_bit/1), + jit_regs:regs_to_mask( + ?AVAILABLE_REGS -- SetArgsPushStackAvailableArgs, fun reg_bit/1 + ) ), stream = Stream1 }, @@ -1625,7 +1563,7 @@ call_func_ptr( [Arg5, Args6] -> set_stack_args(State1, Arg5, Args6) end, - SetArgsRegsOnlyAvailableArgs = mask_to_list(State2#state.available_regs), + SetArgsRegsOnlyAvailableArgs = mask_to_list(jit_regs:available_regs(State2#state.regs)), ParameterRegs = parameter_regs(RegArgs0), {Stream3, SetArgsAvailableRegs, FuncPtrReg, RegArgs} = case FuncPtrTuple of @@ -1678,8 +1616,11 @@ call_func_ptr( end, State3 = State2#state{ - available_regs = jit_regs:regs_to_mask(SetArgsAvailableRegs, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask(?AVAILABLE_REGS -- SetArgsAvailableRegs, fun reg_bit/1), + regs = jit_regs:set_masks( + State2#state.regs, + jit_regs:regs_to_mask(SetArgsAvailableRegs, fun reg_bit/1), + jit_regs:regs_to_mask(?AVAILABLE_REGS -- SetArgsAvailableRegs, fun reg_bit/1) + ), stream = Stream3 }, @@ -1751,9 +1692,11 @@ call_func_ptr( { State4#state{ stream = Stream8, - available_regs = jit_regs:regs_to_mask(AvailableRegs3, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask(UsedRegs2, fun reg_bit/1), - regs = Regs1 + regs = jit_regs:set_masks( + Regs1, + jit_regs:regs_to_mask(AvailableRegs3, fun reg_bit/1), + jit_regs:regs_to_mask(UsedRegs2, fun reg_bit/1) + ) }, ResultReg }. @@ -1848,11 +1791,12 @@ set_registers_args(State0, Args, StackOffset) -> set_registers_args(State0, Args, ParamRegs, StackOffset). set_registers_args( - #state{used_regs = UsedRegsMask} = State0, + #state{regs = Regs0} = State0, Args, ParamRegs, StackOffset ) -> + UsedRegsMask = jit_regs:used_regs(Regs0), ArgsRegs = args_regs(Args), UsedRegsList = mask_to_list(UsedRegsMask), AvailableScratchGP = ((?SCRATCH_REGS -- ParamRegs) -- ArgsRegs) -- UsedRegsList, @@ -1873,8 +1817,11 @@ set_registers_args( FinalUsed = ParamRegs ++ (NewUsedRegsList -- ParamRegs), State1#state{ stream = Stream1, - available_regs = jit_regs:regs_to_mask(FinalAvail, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask(FinalUsed, fun reg_bit/1) + regs = jit_regs:set_masks( + State1#state.regs, + jit_regs:regs_to_mask(FinalAvail, fun reg_bit/1), + jit_regs:regs_to_mask(FinalUsed, fun reg_bit/1) + ) }. parameter_regs(Args) -> @@ -1991,11 +1938,12 @@ set_registers_args1( Stream1 = StreamModule:append(Stream0, I), State#state{stream = Stream1}; set_registers_args1( - #state{available_regs = AvailRegs} = State, + #state{regs = Regs0} = State, {y_reg, X}, Reg, _StackOffset ) -> + AvailRegs = jit_regs:available_regs(Regs0), ldr_y_reg(State, Reg, X, AvailRegs); set_registers_args1( #state{stream_module = StreamModule, stream = Stream0} = State, ArgReg, Reg, _StackOffset @@ -2049,10 +1997,11 @@ move_to_vm_register_emit(State0, Src, {ptr, Reg}) when is_atom(Src) -> Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), State0#state{stream = Stream1}; move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, Src, {y_reg, Y} + #state{regs = Regs0} = State0, Src, {y_reg, Y} ) when is_atom(Src) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp1)), Code = str_y_reg(Src, Y, Temp1, AT), @@ -2067,10 +2016,11 @@ move_to_vm_register_emit( State0#state{stream = Stream1, regs = Regs2}; % Source is an integer to y_reg (optimized: ldr first, then movs) move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, N, {y_reg, Y} + #state{regs = Regs0} = State0, N, {y_reg, Y} ) when is_integer(N), N >= 0, N =< 255 -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail1 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail1), @@ -2087,63 +2037,75 @@ move_to_vm_register_emit( end, State0#state{stream = Stream1, regs = Regs2}; % Source is an integer (0-255 for movs, negative values need different handling) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N), N >= 0, N =< 255 -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_arm32_asm:mov(al, Temp, N), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; + with_temp( + State0, + fun(StateAT, Temp, _AT) -> + I1 = jit_arm32_asm:mov(al, Temp, N), + Stream1 = (StateAT#state.stream_module):append(StateAT#state.stream, I1), + move_to_vm_register(StateAT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); %% Handle large values using simple literal pool (branch-over pattern) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, N), - State2 = move_to_vm_register(State1, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State2#state.regs, Temp), - State2#state{available_regs = AR0, regs = Regs1}; + with_temp( + State0, + fun(StateAT, Temp, _AT) -> + State1 = mov_immediate(StateAT, Temp, N), + move_to_vm_register(State1, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); % Source is a VM register -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, extra}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_arm32_asm:ldr(al, Temp, ?X_REG(?MAX_REG)), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, X}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_arm32_asm:ldr(al, Temp, ?X_REG(X)), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {ptr, Reg}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_arm32_asm:ldr(al, Temp, {Reg, 0}), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {y_reg, Y}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - State0a = ldr_y_reg(State0#state{available_regs = AT}, Temp, Y, AT), - State1 = move_to_vm_register(State0a, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; +move_to_vm_register_emit(State0, {x_reg, extra}, Dest) -> + with_temp( + State0, + fun(StateAT, Temp, _AT) -> + I1 = jit_arm32_asm:ldr(al, Temp, ?X_REG(?MAX_REG)), + Stream1 = (StateAT#state.stream_module):append(StateAT#state.stream, I1), + move_to_vm_register(StateAT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, ?MAX_REG}) end + ); +move_to_vm_register_emit(State0, {x_reg, X}, Dest) -> + with_temp( + State0, + fun(StateAT, Temp, _AT) -> + I1 = jit_arm32_asm:ldr(al, Temp, ?X_REG(X)), + Stream1 = (StateAT#state.stream_module):append(StateAT#state.stream, I1), + move_to_vm_register(StateAT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, X}) end + ); +move_to_vm_register_emit(State0, {ptr, Reg}, Dest) -> + with_temp( + State0, + fun(StateAT, Temp, _AT) -> + I1 = jit_arm32_asm:ldr(al, Temp, {Reg, 0}), + Stream1 = (StateAT#state.stream_module):append(StateAT#state.stream, I1), + move_to_vm_register(StateAT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:invalidate_reg(Regs, Temp) end + ); +move_to_vm_register_emit(State0, {y_reg, Y}, Dest) -> + with_temp( + State0, + fun(StateAT, Temp, AT) -> + State0a = ldr_y_reg(StateAT, Temp, Y, AT), + move_to_vm_register(State0a, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {y_reg, Y}) end + ); % term_to_float move_to_vm_register_emit( #state{ stream_module = StreamModule, - available_regs = Avail, + regs = Regs0, stream = Stream0, variant = Variant } = @@ -2151,6 +2113,7 @@ move_to_vm_register_emit( {free, {ptr, Reg, 1}}, {fp_reg, F} ) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), I1 = jit_arm32_asm:ldr(al, Temp1, ?FP_REGS), @@ -2172,6 +2135,19 @@ move_to_vm_register_emit( Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp1), Temp2), State1#state{stream = Stream1, regs = Regs1}. +-spec with_temp( + state(), + fun((state(), arm32_register(), non_neg_integer()) -> state()), + fun((jit_regs:regs(), arm32_register()) -> jit_regs:regs()) +) -> state(). +with_temp(#state{regs = Regs0} = State0, EmitFun, ContentsFun) -> + AR0 = jit_regs:available_regs(Regs0), + Temp = first_avail(AR0), + AT = AR0 band (bnot reg_bit(Temp)), + State1 = EmitFun(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, AT), + Regs1 = jit_regs:set_available_regs(ContentsFun(State1#state.regs, Temp), AR0), + State1#state{regs = Regs1}. + %%----------------------------------------------------------------------------- %% @doc Emit a move of an array element (reg[x]) to a vm or a native register. %% @end @@ -2188,12 +2164,13 @@ move_to_vm_register_emit( vm_register() | arm32_register() ) -> state(). move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) andalso Index * 4 =< 4095 -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:ldr(al, Temp, {Reg, Index * 4}), I2 = jit_arm32_asm:str(al, Temp, ?X_REG(X)), @@ -2202,12 +2179,13 @@ move_array_element( Regs2 = jit_regs:invalidate_reg(Regs1, Temp), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, available_regs = Avail} = + #state{stream_module = StreamModule, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), % For large offsets, use max offset (4092) in ldr + remainder in temp register @@ -2228,12 +2206,13 @@ move_array_element( Regs2 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs1, Temp1), Temp2), State1#state{stream = Stream2, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) andalso Index * 4 =< 4095 -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:ldr(al, Temp, {Reg, Index * 4}), I2 = jit_arm32_asm:str(al, Temp, {Dest, 0}), @@ -2241,12 +2220,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, available_regs = Avail} = + #state{stream_module = StreamModule, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % For large offsets, use max offset (4092) in ldr + remainder in temp register Offset = Index * 4, @@ -2262,12 +2242,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), State1#state{stream = Stream2, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) andalso Index * 4 =< 4095 -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail1 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail1), @@ -2281,13 +2262,14 @@ move_array_element( State#state{stream = Stream1, regs = Regs2}; move_array_element( #state{ - stream_module = StreamModule, available_regs = Avail + stream_module = StreamModule, regs = Regs0 } = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail1 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail1), @@ -2307,12 +2289,13 @@ move_array_element( Regs2 = jit_regs:invalidate_vm_loc(Regs1, {y_reg, Y}), State1#state{stream = Stream2, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Index, {y_reg, Y} ) when is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), I1 = jit_arm32_asm:ldr(al, Reg, {Reg, Index * 4}), @@ -2333,8 +2316,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2345,23 +2326,17 @@ move_array_element( I2 = jit_arm32_asm:ldr(al, IndexReg, {Reg, IndexReg}), I3 = jit_arm32_asm:str(al, IndexReg, ?X_REG(X)), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_vm_loc(Regs0, {x_reg, X}), Regs2 = jit_regs:invalidate_reg(Regs1, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs2 + regs = jit_regs:free_reg(Regs2, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2372,28 +2347,23 @@ move_array_element( I2 = jit_arm32_asm:ldr(al, IndexReg, {Reg, IndexReg}), I3 = jit_arm32_asm:str(al, IndexReg, {PtrReg, 0}), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_reg(Regs0, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs1 + regs = jit_regs:free_reg(Regs1, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, {free, IndexReg}, {y_reg, Y} ) when is_atom(IndexReg) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), I1 = jit_arm32_asm:lsl(al, IndexReg, IndexReg, 2), I2 = jit_arm32_asm:ldr(al, IndexReg, {Reg, IndexReg}), Temp = first_avail(AvailableRegs0), @@ -2402,18 +2372,14 @@ move_array_element( Code = str_y_reg(IndexReg, Y, Temp, AT), I3 = Code, Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append( Stream0, <> ), Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Temp), IndexReg), Regs2 = jit_regs:invalidate_vm_loc(Regs1, {y_reg, Y}), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs2 + regs = jit_regs:free_reg(Regs2, Bit) }. %% @doc move reg[x] to a vm or native register @@ -2435,11 +2401,12 @@ get_array_element( get_array_element( #state{ stream_module = StreamModule, - available_regs = Avail + regs = Regs0 } = State, {free, Reg}, Index ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % For large offsets, split into ldr immediate (max 4092) + remainder in temp register Offset = Index * 4, @@ -2455,13 +2422,12 @@ get_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, Index ) when Index * 4 =< 4095 -> + Avail = jit_regs:available_regs(Regs0), ElemReg = first_avail(Avail), ElemBit = reg_bit(ElemReg), I1 = jit_arm32_asm:ldr(al, ElemReg, {Reg, Index * 4}), @@ -2470,21 +2436,19 @@ get_array_element( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ElemBit), - used_regs = UsedRegs0 bor ElemBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ElemBit) }, ElemReg }; get_array_element( #state{ stream_module = StreamModule, - available_regs = Avail, - used_regs = UsedRegs0 + regs = Regs0 } = State, Reg, Index ) -> + Avail = jit_regs:available_regs(Regs0), ElemReg = first_avail(Avail), ElemBit = reg_bit(ElemReg), Avail1 = Avail band (bnot ElemBit), @@ -2502,9 +2466,7 @@ get_array_element( { State1#state{ stream = Stream2, - available_regs = Avail1, - used_regs = UsedRegs0 bor ElemBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ElemBit) }, ElemReg }. @@ -2523,11 +2485,12 @@ move_to_array_element( Stream1 = StreamModule:append(Stream0, I1), State0#state{stream = Stream1}; move_to_array_element( - #state{stream_module = StreamModule, available_regs = Avail} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, ValueReg, Reg, Index ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % For large offsets, split into str immediate (max 4092) + remainder in temp register Offset = Index * 4, @@ -2541,12 +2504,13 @@ move_to_array_element( Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), State1#state{stream = Stream2, regs = Regs1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, ValueReg, Reg, IndexReg ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(Reg) andalso ?IS_GPR(IndexReg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:mov(al, Temp, IndexReg), I2 = jit_arm32_asm:lsl(al, Temp, Temp, 2), @@ -2573,12 +2537,13 @@ move_to_array_element( ) when is_integer(IndexReg) andalso is_integer(Offset) -> move_to_array_element(State, Value, BaseReg, IndexReg + Offset); move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, ValueReg, BaseReg, IndexReg, Offset ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:add(al, Temp, IndexReg, Offset), I2 = jit_arm32_asm:lsl(al, Temp, Temp, 2), @@ -2593,7 +2558,7 @@ move_to_array_element( Offset ) -> {State1, ValueReg} = copy_to_native_register(State0, Value), - Temp = first_avail(State1#state.available_regs), + Temp = first_avail(jit_regs:available_regs(State1#state.regs)), I1 = jit_arm32_asm:add(al, Temp, IndexReg, Offset), I2 = jit_arm32_asm:lsl(al, Temp, Temp, 2), I3 = jit_arm32_asm:str(al, ValueReg, {BaseReg, Temp}), @@ -2611,16 +2576,17 @@ move_to_native_register(#state{regs = Regs} = State, Value) -> case Contents =/= unknown andalso jit_regs:find_reg_with_contents(Regs, Contents) of {ok, CachedReg} -> Bit = reg_bit(CachedReg), - case State#state.used_regs band Bit of + Used = jit_regs:used_regs(Regs), + case Used band Bit of 0 -> - case State#state.available_regs band Bit of + Avail = jit_regs:available_regs(Regs), + case Avail band Bit of 0 -> move_to_native_register_emit(State, Value, Contents); _ -> { State#state{ - used_regs = State#state.used_regs bor Bit, - available_regs = State#state.available_regs band (bnot Bit) + regs = jit_regs:alloc_reg(Regs, Bit) }, CachedReg } @@ -2636,13 +2602,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, cp, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_arm32_asm:ldr(al, Reg, ?CP), @@ -2651,9 +2616,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2667,56 +2630,40 @@ move_to_native_register_emit( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {State#state{stream = Stream1, regs = Regs1}, Reg}; move_to_native_register_emit( - #state{ - available_regs = Avail, - used_regs = Used, - regs = Regs0 - } = State0, + #state{regs = Regs0} = State0, Imm, Contents ) when is_integer(Imm) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), Regs1 = jit_regs:set_contents(Regs0, Reg, Contents), - State1 = State0#state{ - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 - }, + Regs2 = jit_regs:alloc_reg(Regs1, Bit), + State1 = State0#state{regs = Regs2}, {move_to_native_register(State1, Imm, Reg), Reg}; move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, extra}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_arm32_asm:ldr(al, Reg, ?X_REG(?MAX_REG)), Stream1 = StreamModule:append(Stream0, I1), Regs1 = jit_regs:set_contents(Regs0, Reg, Contents), - { - State#state{ - stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 - }, - Reg - }; + Regs2 = jit_regs:alloc_reg(Regs1, Bit), + {State#state{stream = Stream1, regs = Regs2}, Reg}; move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, X}, @@ -2724,52 +2671,42 @@ move_to_native_register_emit( ) when X < ?MAX_REG -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_arm32_asm:ldr(al, Reg, ?X_REG(X)), Stream1 = StreamModule:append(Stream0, I1), Regs1 = jit_regs:set_contents(Regs0, Reg, Contents), - { - State#state{ - stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 - }, - Reg - }; + Regs2 = jit_regs:alloc_reg(Regs1, Bit), + {State#state{stream = Stream1, regs = Regs2}, Reg}; move_to_native_register_emit( - #state{ - available_regs = Avail, - used_regs = Used - } = State, + #state{regs = Regs0} = State, {y_reg, Y}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), AvailT = Avail band (bnot Bit), State1 = ldr_y_reg( - State#state{available_regs = AvailT, used_regs = Used bor Bit}, + State#state{regs = jit_regs:alloc_reg(Regs0, Bit)}, Reg, Y, AvailT ), Regs1 = jit_regs:set_contents(State1#state.regs, Reg, Contents), - { - State1#state{regs = Regs1}, - Reg - }; + {State1#state{regs = Regs1}, Reg}; move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used + regs = Regs0 } = State, {fp_reg, F}, _Contents ) -> + Avail = jit_regs:available_regs(Regs0), + Used = jit_regs:used_regs(Regs0), RegA = first_avail(Avail), BitA = reg_bit(RegA), Avail1 = Avail band (bnot BitA), @@ -2781,10 +2718,8 @@ move_to_native_register_emit( I3 = jit_arm32_asm:ldr(al, RegB, {RegB, F * 8 + 4}), Code = <>, Stream1 = StreamModule:append(Stream0, Code), - { - State#state{stream = Stream1, available_regs = AvailT, used_regs = Used bor BitA bor BitB}, - {fp, RegA, RegB} - }. + Regs1 = jit_regs:set_masks(Regs0, AvailT, Used bor BitA bor BitB), + {State#state{stream = Stream1, regs = Regs1}, {fp, RegA, RegB}}. -spec move_to_native_register(state(), value(), arm32_register()) -> state(). move_to_native_register( @@ -2826,14 +2761,15 @@ move_to_native_register( Regs1 = jit_regs:set_contents(Regs0, RegDst, {x_reg, X}), State#state{stream = Stream1, regs = Regs1}; move_to_native_register( - #state{available_regs = AT} = State, + #state{regs = Regs0} = State, {y_reg, Y}, RegDst ) -> + AT = jit_regs:available_regs(Regs0), State1 = ldr_y_reg(State, RegDst, Y, AT), - #state{regs = Regs0} = State1, - Regs1 = jit_regs:set_contents(Regs0, RegDst, {y_reg, Y}), - State1#state{regs = Regs1}; + #state{regs = Regs1} = State1, + Regs2 = jit_regs:set_contents(Regs1, RegDst, {y_reg, Y}), + State1#state{regs = Regs2}; move_to_native_register( #state{ stream_module = StreamModule, @@ -2854,24 +2790,22 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, Reg ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = jit_arm32_asm:mov(al, SaveReg, Reg), Stream1 = StreamModule:append(Stream0, I1), SrcContents = jit_regs:get_contents(Regs0, Reg), Regs1 = jit_regs:set_contents(Regs0, SaveReg, SrcContents), + Regs2 = jit_regs:alloc_reg(Regs1, SaveBit), { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = Regs2 }, SaveReg }; @@ -2879,23 +2813,21 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {ptr, Reg} ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = jit_arm32_asm:ldr(al, SaveReg, {Reg, 0}), Stream1 = StreamModule:append(Stream0, I1), Regs1 = jit_regs:invalidate_reg(Regs0, SaveReg), + Regs2 = jit_regs:alloc_reg(Regs1, SaveBit), { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = Regs2 }, SaveReg }; @@ -2903,21 +2835,27 @@ copy_to_native_register(State, Reg) -> move_to_native_register(State, Reg). move_to_cp( - #state{available_regs = Avail} = State, + #state{regs = Regs0} = State, {y_reg, Y} ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), + % The saved cp spans two slots: y[Y] = offset word (-> ?CP), y[Y+1] = Module* + % (-> ?CP_MODULE). Copy both into ctx->cp. State1 = ldr_y_reg(State, Reg, Y, AvailT), - I2 = jit_arm32_asm:str(al, Reg, ?CP), - Stream1 = (State1#state.stream_module):append(State1#state.stream, I2), - State1#state{stream = Stream1}. + SM = State1#state.stream_module, + Stream1 = SM:append(State1#state.stream, jit_arm32_asm:str(al, Reg, ?CP)), + State2 = ldr_y_reg(State1#state{stream = Stream1}, Reg, Y + 1, AvailT), + Stream2 = SM:append(State2#state.stream, jit_arm32_asm:str(al, Reg, ?CP_MODULE)), + State2#state{stream = Stream2}. increment_sp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Offset ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), I1 = jit_arm32_asm:ldr(al, Reg, ?Y_REGS), I2 = jit_arm32_asm:add(al, Reg, Reg, Offset * 4), @@ -2932,10 +2870,11 @@ set_continuation_to_label( stream_module = StreamModule, stream = Stream0, offset = JumpTableOffset, - available_regs = Avail + regs = Regs0 } = State, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), % Calculate jump table entry offset @@ -2970,11 +2909,11 @@ set_continuation_to_offset( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempJitState = first_avail(Avail band (bnot reg_bit(Temp))), OffsetRef = make_ref(), @@ -3013,11 +2952,10 @@ get_module_index( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), Avail1 = Avail band (bnot RegBit), @@ -3030,16 +2968,57 @@ get_module_index( Stream1 = StreamModule:append(Stream0, Code), Regs1 = jit_regs:invalidate_reg(Regs0, TempJitState), Regs2 = jit_regs:set_contents(Regs1, Reg, module_index), + Regs3 = jit_regs:alloc_reg(Regs2, RegBit), { State#state{ stream = Stream1, - available_regs = Avail1, - used_regs = UsedRegs0 bor RegBit, - regs = Regs2 + regs = Regs3 }, Reg }. +%% @doc Load the current module pointer (jit_state->module) into a register. +get_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + Avail1 = Avail band (bnot RegBit), + TempJitState = first_avail(Avail1), + I1a = jit_arm32_asm:ldr(al, TempJitState, {sp, ?STACK_OFFSET_JITSTATE}), + I1b = jit_arm32_asm:ldr(al, Reg, ?JITSTATE_MODULE(TempJitState)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, TempJitState), Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the Module pointer stored in ctx->cp (?CP_MODULE) into a register. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_arm32_asm:ldr(al, Reg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the offset word (offset << 2) stored in ctx->cp (?CP) into a register. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_arm32_asm:ldr(al, Reg, ?CP), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. %% @doc Perform an AND of a register with an immediate. %% JIT currently calls this with two values: ?TERM_PRIMARY_CLEAR_MASK (-4) to %% clear bits and ?TERM_BOXED_TAG_MASK (0x3F). We can avoid any literal pool @@ -3066,81 +3045,98 @@ and_( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {State0#state{stream = Stream1, regs = Regs1}, Reg}; and_( - #state{stream_module = StreamModule, available_regs = Avail} = State0, - {free, Reg}, - Val -) when Avail =/= 0 andalso Val < 0 andalso Val >= -256 -> - Temp = first_avail(Avail), - TempBit = reg_bit(Temp), - AT = Avail band (bnot TempBit), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, bnot (Val)), - Stream1 = State1#state.stream, - I = jit_arm32_asm:bic(al, Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), - {State1#state{available_regs = AT bor TempBit, stream = Stream2, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = Avail} = State0, - {free, Reg}, - Val -) when Avail =/= 0 -> - Temp = first_avail(Avail), - TempBit = reg_bit(Temp), - AT = Avail band (bnot TempBit), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_arm32_asm:and_(al, Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), - {State1#state{available_regs = AT bor TempBit, stream = Stream2, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = 0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val ) when Val < 0 andalso Val >= -256 -> - % No available registers, use r0 as temp and save it to r12 - Stream0 = State0#state.stream, - % Save r0 to r12 - Save = jit_arm32_asm:mov(al, ?IP_REG, r0), - Stream1 = StreamModule:append(Stream0, Save), - % Load immediate value into r0 - State1 = mov_immediate(State0#state{stream = Stream1}, r0, bnot (Val)), - Stream2 = State1#state.stream, - % Perform BIC operation - I = jit_arm32_asm:bic(al, Reg, Reg, r0), - Stream3 = StreamModule:append(Stream2, I), - % Restore r0 from r12 - Restore = jit_arm32_asm:mov(al, r0, ?IP_REG), - Stream4 = StreamModule:append(Stream3, Restore), - Regs1 = jit_regs:invalidate_reg(State0#state.regs, Reg), - {State0#state{stream = Stream4, regs = Regs1}, Reg}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + % No available registers, use r0 as temp and save it to r12 + Stream0 = State0#state.stream, + % Save r0 to r12 + Save = jit_arm32_asm:mov(al, ?IP_REG, r0), + Stream1 = StreamModule:append(Stream0, Save), + % Load immediate value into r0 + State1 = mov_immediate(State0#state{stream = Stream1}, r0, bnot (Val)), + Stream2 = State1#state.stream, + % Perform BIC operation + I = jit_arm32_asm:bic(al, Reg, Reg, r0), + Stream3 = StreamModule:append(Stream2, I), + % Restore r0 from r12 + Restore = jit_arm32_asm:mov(al, r0, ?IP_REG), + Stream4 = StreamModule:append(Stream3, Restore), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + {State0#state{stream = Stream4, regs = Regs1}, Reg}; + _ -> + Temp = first_avail(Avail), + TempBit = reg_bit(Temp), + AT = Avail band (bnot TempBit), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, bnot (Val) + ), + Stream1 = State1#state.stream, + I = jit_arm32_asm:bic(al, Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, AT bor TempBit) + }, + Reg + } + end; and_( - #state{stream_module = StreamModule, available_regs = 0, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val ) -> - % No available registers, use r0 as temp and save it to r12 - Stream0 = State0#state.stream, - % Save r0 to r12 - Save = jit_arm32_asm:mov(al, ?IP_REG, r0), - Stream1 = StreamModule:append(Stream0, Save), - % Load immediate value into r0 - State1 = mov_immediate(State0#state{stream = Stream1}, r0, Val), - Stream2 = State1#state.stream, - % Perform AND operation - I = jit_arm32_asm:and_(al, Reg, Reg, r0), - Stream3 = StreamModule:append(Stream2, I), - % Restore r0 from r12 - Restore = jit_arm32_asm:mov(al, r0, ?IP_REG), - Stream4 = StreamModule:append(Stream3, Restore), - Regs1 = jit_regs:invalidate_reg(Regs0, Reg), - {State0#state{stream = Stream4, regs = Regs1}, Reg}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + % No available registers, use r0 as temp and save it to r12 + Stream0 = State0#state.stream, + % Save r0 to r12 + Save = jit_arm32_asm:mov(al, ?IP_REG, r0), + Stream1 = StreamModule:append(Stream0, Save), + % Load immediate value into r0 + State1 = mov_immediate(State0#state{stream = Stream1}, r0, Val), + Stream2 = State1#state.stream, + % Perform AND operation + I = jit_arm32_asm:and_(al, Reg, Reg, r0), + Stream3 = StreamModule:append(Stream2, I), + % Restore r0 from r12 + Restore = jit_arm32_asm:mov(al, r0, ?IP_REG), + Stream4 = StreamModule:append(Stream3, Restore), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + {State0#state{stream = Stream4, regs = Regs1}, Reg}; + _ -> + Temp = first_avail(Avail), + TempBit = reg_bit(Temp), + AT = Avail band (bnot TempBit), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_arm32_asm:and_(al, Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, AT bor TempBit) + }, + Reg + } + end; and_( - #state{stream_module = StreamModule, available_regs = Avail, used_regs = UR, regs = Regs0} = + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, ?TERM_PRIMARY_CLEAR_MASK ) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I1 = jit_arm32_asm:lsr(al, ResultReg, Reg, 2), @@ -3150,9 +3146,7 @@ and_( { State0#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -3165,18 +3159,21 @@ or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; or_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_arm32_asm:orr(al, Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. xor_( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Reg, SrcReg @@ -3188,18 +3185,21 @@ xor_( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; xor_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_arm32_asm:eor(al, Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. add(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Reg, Val) when (Val >= 0 andalso Val =< 255) orelse is_atom(Val) @@ -3208,15 +3208,18 @@ add(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; -add(#state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, Reg, Val) -> +add(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_arm32_asm:add(al, Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(Regs0, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. mov_immediate( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Val @@ -3346,21 +3349,25 @@ sub(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I1), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State#state{stream = Stream1, regs = Regs1}; -sub(#state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, Reg, Val) -> +sub(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_arm32_asm:sub(al, Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(Regs0, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. mul(State, _Reg, 1) -> State; mul(State, Reg, 2) -> shift_left(State, Reg, 1); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> +mul(#state{regs = Regs0} = State, Reg, 3) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:lsl(al, Temp, Reg, 1), I2 = jit_arm32_asm:add(al, Reg, Temp, Reg), @@ -3369,7 +3376,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 4) -> shift_left(State, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> +mul(#state{regs = Regs0} = State, Reg, 5) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:lsl(al, Temp, Reg, 2), I2 = jit_arm32_asm:add(al, Reg, Temp, Reg), @@ -3379,7 +3387,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> mul(State0, Reg, 6) -> State1 = mul(State0, Reg, 3), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> +mul(#state{regs = Regs0} = State, Reg, 7) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:lsl(al, Temp, Reg, 3), I2 = jit_arm32_asm:sub(al, Reg, Temp, Reg), @@ -3388,7 +3397,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 8) -> shift_left(State, Reg, 3); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> +mul(#state{regs = Regs0} = State, Reg, 9) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:lsl(al, Temp, Reg, 3), I2 = jit_arm32_asm:add(al, Reg, Temp, Reg), @@ -3398,7 +3408,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> mul(State0, Reg, 10) -> State1 = mul(State0, Reg, 5), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 15) -> +mul(#state{regs = Regs0} = State, Reg, 15) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:lsl(al, Temp, Reg, 4), I2 = jit_arm32_asm:sub(al, Reg, Temp, Reg), @@ -3412,20 +3423,26 @@ mul(State, Reg, 32) -> mul(State, Reg, 64) -> shift_left(State, Reg, 6); mul( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempBit = reg_bit(Temp), AT = Avail band (bnot TempBit), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_arm32_asm:mul(al, Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Temp), Reg), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp), Reg), State1#state{ - stream = Stream2, available_regs = State1#state.available_regs bor TempBit, regs = Regs1 + stream = Stream2, + regs = jit_regs:set_available_regs( + Regs1, jit_regs:available_regs(State1#state.regs) bor TempBit + ) }; mul( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DestReg, SrcReg @@ -3442,9 +3459,10 @@ mul( -spec decrement_reductions_and_maybe_schedule_next(state()) -> state(). decrement_reductions_and_maybe_schedule_next( #state{ - stream_module = StreamModule, stream = Stream0, available_regs = Avail + stream_module = StreamModule, stream = Stream0, regs = Regs0 } = State0 ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempJitState = first_avail(Avail band (bnot reg_bit(Temp))), % Load jit_state pointer from stack @@ -3485,9 +3503,8 @@ decrement_reductions_and_maybe_schedule_next( Stream5 = StreamModule:replace( Stream4, BNEOffset, <> ), - State3 = merge_used_regs(State2#state{stream = Stream5}, State1#state.used_regs), %% schedule_next clobbers caller-saved regs; invalidate cache at continuation. - State3#state{regs = jit_regs:invalidate_all(State1#state.regs)}. + State2#state{stream = Stream5, regs = jit_regs:invalidate_all(State1#state.regs)}. -spec call_or_schedule_next(state(), non_neg_integer()) -> state(). call_or_schedule_next(State0, Label) -> @@ -3499,10 +3516,11 @@ call_only_or_schedule_next( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempJitState = first_avail(Avail band (bnot reg_bit(Temp))), % Load jit_state pointer from stack @@ -3552,29 +3570,24 @@ call_primitive_with_cp(State0, Primitive, Args) -> -spec set_cp(state()) -> {state(), non_neg_integer(), arm32_register()}. set_cp(State0) -> - % get module index (dynamically) - { - #state{stream_module = StreamModule, stream = Stream0, available_regs = AvailRegs} = State1, - Reg - } = get_module_index( - State0 - ), - % Get a temporary register from available registers + % cp is two words: store the Module pointer (jit_state->module) at ?CP_MODULE, + % and the return offset << 2 at ?CP (patched by rewrite_cp_offset below). + {#state{stream_module = StreamModule, stream = Stream0} = State1, ModReg} = + get_module(State0), + IModStore = jit_arm32_asm:str(al, ModReg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + AvailRegs = jit_regs:available_regs(State2#state.regs), + % Get a temporary register to hold the offset value TempReg = first_avail(AvailRegs), - - Offset = StreamModule:offset(Stream0), - % build cp with module_index << 24 - I1 = jit_arm32_asm:lsl(al, Reg, Reg, 24), - % Placeholder for offset load instruction - I2 = <<16#FFFFFFFF:32>>, - MOVOffset = Offset + byte_size(I1), - % OR the module index with the offset (loaded in temp register) - I3 = jit_arm32_asm:orr(al, Reg, Reg, TempReg), - I4 = jit_arm32_asm:str(al, Reg, ?CP), - Code = <>, - Stream1 = StreamModule:append(Stream0, Code), - State2 = State1#state{stream = Stream1}, - State3 = free_native_register(State2, Reg), + Offset = StreamModule:offset(Stream1), + % Placeholder for the offset load instruction (patched by rewrite_cp_offset) + I1 = <<16#FFFFFFFF:32>>, + MOVOffset = Offset, + I2 = jit_arm32_asm:str(al, TempReg, ?CP), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, {State3, MOVOffset, TempReg}. -spec rewrite_cp_offset(state(), non_neg_integer(), arm32_register()) -> state(). @@ -3614,10 +3627,11 @@ rewrite_cp_offset( end. set_bs( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, TermReg ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_arm32_asm:str(al, TermReg, ?BS), I2 = jit_arm32_asm:mov(al, Temp, 0), @@ -3759,69 +3773,6 @@ reg_bit(r13) -> ?REG_BIT_R13; reg_bit(r14) -> ?REG_BIT_R14; reg_bit(r15) -> ?REG_BIT_R15. -%% first_avail returns the first available register from a bitmask. -%% Order matches ?AVAILABLE_REGS = [r11, r10, r9, r8, r7, r6, r5, r4, r3, r1] -first_avail(Mask) when Mask band ?REG_BIT_R11 =/= 0 -> r11; -first_avail(Mask) when Mask band ?REG_BIT_R10 =/= 0 -> r10; -first_avail(Mask) when Mask band ?REG_BIT_R9 =/= 0 -> r9; -first_avail(Mask) when Mask band ?REG_BIT_R8 =/= 0 -> r8; -first_avail(Mask) when Mask band ?REG_BIT_R7 =/= 0 -> r7; -first_avail(Mask) when Mask band ?REG_BIT_R6 =/= 0 -> r6; -first_avail(Mask) when Mask band ?REG_BIT_R5 =/= 0 -> r5; -first_avail(Mask) when Mask band ?REG_BIT_R4 =/= 0 -> r4; -first_avail(Mask) when Mask band ?REG_BIT_R3 =/= 0 -> r3; -first_avail(Mask) when Mask band ?REG_BIT_R1 =/= 0 -> r1. - -%% Convert bitmask to list, matching the order of ?AVAILABLE_REGS. -mask_to_list(0) -> []; -mask_to_list(Mask) -> mask_to_list_r11(Mask). - -mask_to_list_r11(Mask) when Mask band ?REG_BIT_R11 =/= 0 -> [r11 | mask_to_list_r10(Mask)]; -mask_to_list_r11(Mask) -> mask_to_list_r10(Mask). -mask_to_list_r10(Mask) when Mask band ?REG_BIT_R10 =/= 0 -> [r10 | mask_to_list_r9(Mask)]; -mask_to_list_r10(Mask) -> mask_to_list_r9(Mask). -mask_to_list_r9(Mask) when Mask band ?REG_BIT_R9 =/= 0 -> [r9 | mask_to_list_r8(Mask)]; -mask_to_list_r9(Mask) -> mask_to_list_r8(Mask). -mask_to_list_r8(Mask) when Mask band ?REG_BIT_R8 =/= 0 -> [r8 | mask_to_list_r7(Mask)]; -mask_to_list_r8(Mask) -> mask_to_list_r7(Mask). -mask_to_list_r7(Mask) when Mask band ?REG_BIT_R7 =/= 0 -> [r7 | mask_to_list_r6(Mask)]; -mask_to_list_r7(Mask) -> mask_to_list_r6(Mask). -mask_to_list_r6(Mask) when Mask band ?REG_BIT_R6 =/= 0 -> [r6 | mask_to_list_r5(Mask)]; -mask_to_list_r6(Mask) -> mask_to_list_r5(Mask). -mask_to_list_r5(Mask) when Mask band ?REG_BIT_R5 =/= 0 -> [r5 | mask_to_list_r4(Mask)]; -mask_to_list_r5(Mask) -> mask_to_list_r4(Mask). -mask_to_list_r4(Mask) when Mask band ?REG_BIT_R4 =/= 0 -> [r4 | mask_to_list_r3(Mask)]; -mask_to_list_r4(Mask) -> mask_to_list_r3(Mask). -mask_to_list_r3(Mask) when Mask band ?REG_BIT_R3 =/= 0 -> [r3 | mask_to_list_r1(Mask)]; -mask_to_list_r3(Mask) -> mask_to_list_r1(Mask). -mask_to_list_r1(Mask) when Mask band ?REG_BIT_R1 =/= 0 -> [r1]; -mask_to_list_r1(_Mask) -> []. - -args_regs(Args) -> - lists:map( - fun - ({free, {ptr, Reg}}) -> Reg; - ({free, Reg}) when is_atom(Reg) -> Reg; - ({free, Imm}) when is_integer(Imm) -> imm; - (offset) -> imm; - (ctx) -> ?CTX_REG; - (jit_state) -> jit_state; - (jit_state_tail_call) -> jit_state; - (stack) -> stack; - (Reg) when is_atom(Reg) -> Reg; - (Imm) when is_integer(Imm) -> imm; - ({ptr, Reg}) -> Reg; - ({x_reg, _}) -> ?CTX_REG; - ({y_reg, _}) -> ?CTX_REG; - ({fp_reg, _}) -> ?CTX_REG; - ({free, {x_reg, _}}) -> ?CTX_REG; - ({free, {y_reg, _}}) -> ?CTX_REG; - ({free, {fp_reg, _}}) -> ?CTX_REG; - ({avm_int64_t, _}) -> imm - end, - Args - ). - %%----------------------------------------------------------------------------- %% @doc Add a label at the current offset. Eventually align it with a nop. %% @end diff --git a/libs/jit/src/jit_armv6m.erl b/libs/jit/src/jit_armv6m.erl index b3555042b1..4d410131c4 100644 --- a/libs/jit/src/jit_armv6m.erl +++ b/libs/jit/src/jit_armv6m.erl @@ -60,6 +60,9 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -163,8 +166,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), branch_type()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), literal_pool :: [{non_neg_integer(), armv6m_register(), non_neg_integer()}], @@ -201,10 +202,12 @@ -define(NATIVE_INTERFACE_REG, r2). -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). +% ?CP holds the low word (offset << 2), ?CP_MODULE holds the high word (Module*). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). % JITSTATE is on stack, accessed via stack offset % These macros now expect a register that contains the jit_state pointer -define(JITSTATE_MODULE(Reg), {Reg, 0}). @@ -306,12 +309,10 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, labels = #{}, variant = Variant, literal_pool = [], - regs = jit_regs:new(), + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0), thumb2 = (Variant band ?JIT_VARIANT_THUMB2) =/= 0 }. @@ -357,69 +358,14 @@ debugger(#state{stream_module = StreamModule, stream = Stream0} = State) -> Stream1 = StreamModule:append(Stream0, jit_armv6m_asm:bkpt(0)), State#state{stream = Stream1}. -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently used native registers. This is used for -%% debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of used registers -%%----------------------------------------------------------------------------- --spec used_regs(state()) -> [armv6m_register()]. -used_regs(#state{used_regs = Used}) -> mask_to_list(Used). - -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently available native scratch registers. This -%% is used for debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of available registers -%%----------------------------------------------------------------------------- --spec available_regs(state()) -> [armv6m_register()]. -available_regs(#state{available_regs = Available}) -> mask_to_list(Available). - -%%----------------------------------------------------------------------------- -%% @doc Free native registers. The passed list of registers can contain -%% registers, pointer to registers or other values that are ignored. -%% @end -%% @param State current backend state -%% @param Regs list of registers or other values -%% @return The updated backend state -%%----------------------------------------------------------------------------- --spec free_native_registers(state(), [value()]) -> state(). -free_native_registers(State, []) -> - State; -free_native_registers(State, [Reg | Rest]) -> - State1 = free_native_register(State, Reg), - free_native_registers(State1, Rest). - --spec free_native_register(state(), value()) -> state(). -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Reg -) when - is_atom(Reg) --> - Bit = reg_bit(Reg), - State#state{ - available_regs = Available0 bor Bit, used_regs = Used0 band (bnot Bit) - }; -free_native_register(State, {ptr, Reg}) -> - free_native_register(State, Reg); -free_native_register(State, _Other) -> - State. - -%%----------------------------------------------------------------------------- -%% @doc Assert that all native scratch registers are available. This is used -%% for debugging and not in production. -%% @end -%% @param State current backend state -%% @return ok -%%----------------------------------------------------------------------------- --spec assert_all_native_free(state()) -> ok. -assert_all_native_free(State) -> - 0 = State#state.used_regs, - ?AVAILABLE_REGS_MASK = State#state.available_regs, - ok. +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1, +%% first_avail/1, mask_to_list/1, args_regs/1, prepare_call_scratch/1) is shared +%% across the register-based backends and flows through jit_regs. +-define(FIRST_AVAIL_REGS, [r7, r6, r5, r4, r3, r1]). +-define(MASK_TO_LIST_REGS, ?FIRST_AVAIL_REGS). +-define(JITSTATE_ARG_REG, jit_state). +-include("jit_backend_regs_impl.hrl"). %%----------------------------------------------------------------------------- %% @doc Emit the jump table at the beginning of the module. Branches will be @@ -677,33 +623,39 @@ load_primitive_ptr(Primitive, TargetReg) -> %%----------------------------------------------------------------------------- -spec call_primitive(state(), non_neg_integer(), [arg()]) -> {state(), armv6m_register()}. call_primitive( + #state{regs = Regs0} = State, + Primitive, + Args +) -> + Available = jit_regs:available_regs(Regs0), + call_primitive_impl(Available, State, Primitive, Args). + +call_primitive_impl(0, State, Primitive, Args) -> + call_func_ptr(State, {primitive, Primitive}, Args); +call_primitive_impl( + Available, #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used + regs = Regs0 } = State, Primitive, Args -) when Available =/= 0 -> +) -> % Use a low register for LDR since ARM Thumb LDR only works with low registers TempReg = first_avail(Available), TempBit = reg_bit(TempReg), PrepCall = load_primitive_ptr(Primitive, TempReg), Stream1 = StreamModule:append(Stream0, PrepCall), + Regs1 = jit_regs:alloc_reg( + jit_regs:invalidate_reg(Regs0, TempReg), + TempBit + ), StateCall = State#state{ stream = Stream1, - available_regs = Available band (bnot TempBit), - used_regs = Used bor TempBit, - regs = jit_regs:invalidate_reg(State#state.regs, TempReg) + regs = Regs1 }, - call_func_ptr(StateCall, {free, TempReg}, Args); -call_primitive( - #state{available_regs = 0} = State, - Primitive, - Args -) -> - call_func_ptr(State, {primitive, Primitive}, Args). + call_func_ptr(StateCall, {free, TempReg}, Args). %%----------------------------------------------------------------------------- %% @doc Emit a jump (call without return) to a primitive with arguments. This @@ -726,23 +678,18 @@ call_primitive_last( % We need a register for the function pointer that should not be used as a parameter % Since we're not returning, we can use all scratch registers except % registers used for parameters - ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), - ArgsRegs = args_regs(Args), - ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), - ArgsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), - ScratchMask = ?AVAILABLE_REGS_MASK band (bnot (ArgsMask bor ParamMask)), - Temp = first_avail(ScratchMask), - TempBit = reg_bit(Temp), - AvailableRegs1 = ScratchMask band (bnot TempBit), - UsedRegs = ?AVAILABLE_REGS_MASK band (bnot AvailableRegs1), + #{temp := Temp, available_mask := AvailableRegs1, used_mask := UsedRegs} = + prepare_call_scratch(Args), PrepCall = load_primitive_ptr(Primitive, Temp), Stream1 = StreamModule:append(Stream0, PrepCall), State1 = State0#state{ stream = Stream1, - available_regs = AvailableRegs1, - used_regs = UsedRegs, - regs = jit_regs:invalidate_reg(State0#state.regs, Temp) + regs = jit_regs:set_masks( + jit_regs:invalidate_reg(State0#state.regs, Temp), + AvailableRegs1, + UsedRegs + ) }, % Preprocess offset special arg @@ -786,9 +733,11 @@ call_primitive_last( tail_call_with_jit_state_registers_only(State2, Temp) end, State5 = State4#state{ - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State4#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State4#state.regs), + ?AVAILABLE_REGS_MASK, + 0 + ) }, flush_literal_pool(State5). @@ -843,8 +792,6 @@ return_if_not_equal_to_ctx( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, {free, Reg} @@ -863,9 +810,7 @@ return_if_not_equal_to_ctx( RegBit = reg_bit(Reg), State#state{ stream = Stream1, - available_regs = AvailableRegs0 bor RegBit, - used_regs = UsedRegs0 band (bnot RegBit), - regs = Regs0 + regs = jit_regs:free_reg(Regs0, RegBit) }. %%----------------------------------------------------------------------------- @@ -911,11 +856,12 @@ jump_to_continuation( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, + regs = Regs0, offset = BaseOffset } = State0, {free, OffsetReg} ) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), % ARM v6-M PIC implementation using one temp register: % 1. Use ADR to get PC into temp register @@ -961,9 +907,11 @@ jump_to_continuation( % Free all registers as this is a terminal instruction State2 = State1#state{ stream = Stream2, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State1#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State1#state.regs), + ?AVAILABLE_REGS_MASK, + 0 + ) }, flush_literal_pool(State2). @@ -979,8 +927,10 @@ branch_to_offset_code(_State, Offset, TargetOffset) when Rel = TargetOffset - Offset, jit_armv6m_asm:b(Rel); branch_to_offset_code( - #state{available_regs = Available}, Offset, TargetOffset -) when Available =/= 0 -> + #state{regs = Regs}, Offset, TargetOffset +) -> + Available = jit_regs:available_regs(Regs), + Available =/= 0 orelse error(no_available_registers), TempReg = first_avail(Available), % Far branch: use register-based sequence, need temporary register if @@ -1014,41 +964,10 @@ branch_to_label_code( BrEntry = {Offset, b_w}, State1 = State0#state{branches = Branches#{Label => [BrEntry | maps:get(Label, Branches, [])]}}, {State1, CodeBlock}; -branch_to_label_code( - #state{available_regs = Available, branches = Branches} = State0, Offset, Label, false -) when Available =/= 0 -> - TempReg = first_avail(Available), - SequenceSize = - if - Offset rem 4 =:= 0 -> - % Aligned - I1 = jit_armv6m_asm:ldr(TempReg, {pc, 4}), - I2 = jit_armv6m_asm:add(TempReg, pc), - I3 = jit_armv6m_asm:bx(TempReg), - % Unaligned : need nop - I4 = jit_armv6m_asm:nop(), - % Placeholder offset - I5 = <<0:32/little>>, - Seq = <>, - byte_size(Seq); - true -> - % Unaligned - I1 = jit_armv6m_asm:ldr(TempReg, {pc, 4}), - I2 = jit_armv6m_asm:add(TempReg, pc), - I3 = jit_armv6m_asm:bx(TempReg), - % Placeholder offset - I4 = <<0:32/little>>, - Seq = <>, - byte_size(Seq) - end, - % Add relocation entry - CodeBlock = binary:copy(<<16#FF>>, SequenceSize), - BrEntry = {Offset, {far_branch, SequenceSize, TempReg}}, - State1 = State0#state{branches = Branches#{Label => [BrEntry | maps:get(Label, Branches, [])]}}, - {State1, CodeBlock}; -branch_to_label_code( - #state{available_regs = 0, branches = Branches} = State0, Offset, Label, false -) -> +branch_to_label_code(#state{regs = Regs} = State0, Offset, Label, false) -> + branch_to_label_code_impl(jit_regs:available_regs(Regs), State0, Offset, Label). + +branch_to_label_code_impl(0, #state{branches = Branches} = State0, Offset, Label) -> SequenceSize = if Offset rem 4 =/= 0 -> @@ -1088,8 +1007,36 @@ branch_to_label_code( BrEntry = {Offset, {far_branch, SequenceSize, ?IP_REG}}, State1 = State0#state{branches = Branches#{Label => [BrEntry | maps:get(Label, Branches, [])]}}, {State1, CodeBlock}; -branch_to_label_code(#state{available_regs = 0}, _Offset, _Label, _LabelLookup) -> - error({no_available_registers, _LabelLookup}). +branch_to_label_code_impl(Available, #state{branches = Branches} = State0, Offset, Label) -> + TempReg = first_avail(Available), + SequenceSize = + if + Offset rem 4 =:= 0 -> + % Aligned + I1 = jit_armv6m_asm:ldr(TempReg, {pc, 4}), + I2 = jit_armv6m_asm:add(TempReg, pc), + I3 = jit_armv6m_asm:bx(TempReg), + % Unaligned : need nop + I4 = jit_armv6m_asm:nop(), + % Placeholder offset + I5 = <<0:32/little>>, + Seq = <>, + byte_size(Seq); + true -> + % Unaligned + I1 = jit_armv6m_asm:ldr(TempReg, {pc, 4}), + I2 = jit_armv6m_asm:add(TempReg, pc), + I3 = jit_armv6m_asm:bx(TempReg), + % Placeholder offset + I4 = <<0:32/little>>, + Seq = <>, + byte_size(Seq) + end, + % Add relocation entry + CodeBlock = binary:copy(<<16#FF>>, SequenceSize), + BrEntry = {Offset, {far_branch, SequenceSize, TempReg}}, + State1 = State0#state{branches = Branches#{Label => [BrEntry | maps:get(Label, Branches, [])]}}, + {State1, CodeBlock}. %%----------------------------------------------------------------------------- %% @doc Emit an if block, i.e. emit a test of a condition and conditionnally @@ -1127,9 +1074,10 @@ if_block( Stream2, Replacements ), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}; + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}; if_block( #state{stream_module = StreamModule, stream = Stream0} = State0, Cond, @@ -1144,9 +1092,10 @@ if_block( BranchOffset = OffsetAfter - (Offset + BranchInstrOffset), NewBranchInstr = rewrite_cond_branch(CC, BranchOffset), Stream3 = StreamModule:replace(Stream2, Offset + BranchInstrOffset, NewBranchInstr), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}. %%----------------------------------------------------------------------------- %% @doc Emit an if else block, i.e. emit a test of a condition and @@ -1184,8 +1133,6 @@ if_else_block( %% Build the else block StateElse = State2#state{ stream = Stream4, - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs, regs = State1#state.regs }, State3 = BlockFalseFn(StateElse), @@ -1195,9 +1142,10 @@ if_else_block( FinalJumpOffset = OffsetFinal - ElseJumpOffset, NewElseJumpInstr = jit_armv6m_asm:b(FinalJumpOffset), Stream6 = StreamModule:replace(Stream5, ElseJumpOffset, NewElseJumpInstr), - State4 = merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs), - MergedRegs = jit_regs:merge(State2#state.regs, State3#state.regs), - State4#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State2#state.regs, State3#state.regs, ?AVAILABLE_REGS_MASK + ), + State3#state{stream = Stream6, regs = MergedRegs}. %% @private %% Regenerate the conditional branch that skips an if-block, given the patched @@ -1240,9 +1188,11 @@ if_block_cond( {State2, le, byte_size(I1)}; %% Handle {Val, '<', Reg} for values > 255, need to load into temp register if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {Val, '<', RegOrTuple} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1297,9 +1247,11 @@ if_block_cond( State2 = State1#state{stream = Stream1}, {State2, CC, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '<', Val} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1320,9 +1272,11 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, CC, Offset1 - Offset0 + byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, available_regs = Available} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {Val, '<', RegOrTuple} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1460,9 +1414,11 @@ if_block_cond( State3 = if_block_free_reg({free, RegB}, State2), {State3, CC, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '==', Val} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Offset0 = StreamModule:offset(Stream0), Reg = @@ -1482,9 +1438,11 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, CC, Offset1 - Offset0 + byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '!=', Val} -) when is_integer(Val), Available =/= 0 -> +) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Offset0 = StreamModule:offset(Stream0), Reg = @@ -1507,10 +1465,12 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {'(bool)', RegOrTuple, '==', false} -) when Available =/= 0 -> +) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1531,10 +1491,12 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {'(bool)', RegOrTuple, '!=', false} -) when Available =/= 0 -> +) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1555,10 +1517,12 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {RegOrTuple, '&', Val, '!=', 0} -) when Available =/= 0 -> +) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -1591,10 +1555,12 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {Reg, '&', 16#F, '!=', 16#F} -) when ?IS_GPR(Reg), Available =/= 0 -> +) when ?IS_GPR(Reg) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), % Special case Reg & ?TERM_IMMED_TAG_MASK != ?TERM_INTEGER_TAG I1 = jit_armv6m_asm:mvns(Temp, Reg), @@ -1628,10 +1594,12 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available + regs = Regs0 } = State0, {Reg, '&', Mask, '!=', Val} -) when ?IS_GPR(Reg), Available =/= 0 -> +) when ?IS_GPR(Reg) -> + Available = jit_regs:available_regs(Regs0), + Available =/= 0 orelse error(no_available_registers), Temp = first_avail(Available), TempBit = reg_bit(Temp), AT = Available band (bnot TempBit), @@ -1640,7 +1608,9 @@ if_block_cond( I1 = jit_armv6m_asm:mov(Temp, Reg), Stream1 = StreamModule:append(Stream0, I1), State1 = State0#state{stream = Stream1}, - {State2, Temp} = and_(State1#state{available_regs = AT}, {free, Temp}, Mask), + {State2, Temp} = and_( + State1#state{regs = jit_regs:set_available_regs(Regs0, AT)}, {free, Temp}, Mask + ), Stream2 = State2#state.stream, % Compare with value I2 = jit_armv6m_asm:cmp(Temp, Val), @@ -1650,9 +1620,10 @@ if_block_cond( ?ASSERT(byte_size(jit_armv6m_asm:bcc(CC, 0)) =:= 2), Stream4 = StreamModule:append(Stream3, <<16#FFFF:16>>), Regs3 = jit_regs:invalidate_reg(State2#state.regs, Temp), - State3 = State2#state{ - stream = Stream4, available_regs = State2#state.available_regs bor TempBit, regs = Regs3 - }, + Regs4 = jit_regs:set_available_regs( + Regs3, jit_regs:available_regs(State2#state.regs) bor TempBit + ), + State3 = State2#state{stream = Stream4, regs = Regs4}, {State3, CC, OffsetAfter - OffsetBefore}; if_block_cond( #state{ @@ -1678,14 +1649,9 @@ if_block_cond( -spec if_block_free_reg(armv6m_register() | {free, armv6m_register()}, state()) -> state(). if_block_free_reg({free, Reg}, State0) -> - #state{available_regs = AvR0, used_regs = UR0} = State0, + #state{regs = Regs0} = State0, Bit = reg_bit(Reg), - AvR1 = AvR0 bor Bit, - UR1 = UR0 band (bnot Bit), - State0#state{ - available_regs = AvR1, - used_regs = UR1 - }; + State0#state{regs = jit_regs:free_reg(Regs0, Bit)}; if_block_free_reg(Reg, State0) when ?IS_GPR(Reg) -> State0. @@ -1702,12 +1668,6 @@ bit_test_optimization(16#F) -> {low_bits_mask, 4}; bit_test_optimization(16#3F) -> {low_bits_mask, 6}; bit_test_optimization(_) -> no_optimization. --spec merge_used_regs(state(), non_neg_integer()) -> state(). -merge_used_regs(#state{used_regs = UR} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - MergedAvail = ?AVAILABLE_REGS_MASK band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. - %%----------------------------------------------------------------------------- %% @doc Emit a shift register right by a fixed number of bits, effectively %% dividing it by 2^Shift @@ -1731,8 +1691,6 @@ shift_right( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1740,16 +1698,18 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I = jit_armv6m_asm:lsrs(ResultReg, Reg, Shift), Stream1 = StreamModule:append(Stream0, I), - Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), + Regs1 = jit_regs:alloc_reg( + jit_regs:invalidate_reg(Regs0, ResultReg), + Bit + ), { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, regs = Regs1 }, ResultReg @@ -1770,8 +1730,6 @@ shift_right_arith( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1779,16 +1737,18 @@ shift_right_arith( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I = jit_armv6m_asm:asrs(ResultReg, Reg, Shift), Stream1 = StreamModule:append(Stream0, I), - Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), + Regs1 = jit_regs:alloc_reg( + jit_regs:invalidate_reg(Regs0, ResultReg), + Bit + ), { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, regs = Regs1 }, ResultReg @@ -1827,12 +1787,13 @@ call_func_ptr( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0Mask, - used_regs = UsedRegs0Mask + regs = Regs0 } = State0, FuncPtrTuple, Args ) -> + AvailableRegs0Mask = jit_regs:available_regs(Regs0), + UsedRegs0Mask = jit_regs:used_regs(Regs0), FreeRegs = lists:flatmap( fun ({free, {ptr, Reg}}) -> [Reg]; @@ -1899,8 +1860,11 @@ call_func_ptr( % and the currently available registers to push values to the stack. SetArgsPushStackMask = (UsedRegs1Mask band (bnot AllArgsRegsMask)) bor AvailableRegs0Mask, State1 = State0#state{ - available_regs = SetArgsPushStackMask, - used_regs = ?AVAILABLE_REGS_MASK band (bnot SetArgsPushStackMask), + regs = jit_regs:set_masks( + Regs0, + SetArgsPushStackMask, + ?AVAILABLE_REGS_MASK band (bnot SetArgsPushStackMask) + ), stream = Stream1 }, State2 = @@ -1910,7 +1874,7 @@ call_func_ptr( [Arg5, Args6] -> set_stack_args(State1, Arg5, Args6) end, - SetArgsMask = State2#state.available_regs, + SetArgsMask = jit_regs:available_regs(State2#state.regs), ParameterRegs = parameter_regs(RegArgs0), ParamMask = regs_to_mask(ParameterRegs), RegArgsRegsMask = regs_to_mask(RegArgsRegs), @@ -1969,8 +1933,11 @@ call_func_ptr( end, State3 = State2#state{ - available_regs = SetArgsAvailMask, - used_regs = ?AVAILABLE_REGS_MASK band (bnot SetArgsAvailMask), + regs = jit_regs:set_masks( + State2#state.regs, + SetArgsAvailMask, + ?AVAILABLE_REGS_MASK band (bnot SetArgsAvailMask) + ), stream = Stream3 }, @@ -2049,12 +2016,14 @@ call_func_ptr( ResultRegBit = reg_bit(ResultReg), AvailableRegs3Mask = (AvailableRegs1Mask band (bnot ResultRegBit)) band ?AVAILABLE_REGS_MASK, - Regs1 = jit_regs:invalidate_all(State0#state.regs), + Regs1 = jit_regs:set_masks( + jit_regs:invalidate_all(State0#state.regs), + AvailableRegs3Mask, + UsedRegs2Mask + ), { State4#state{ stream = Stream8, - available_regs = AvailableRegs3Mask, - used_regs = UsedRegs2Mask, regs = Regs1 }, ResultReg @@ -2143,11 +2112,12 @@ set_registers_args(State0, Args, StackOffset) -> set_registers_args(State0, Args, ParamRegs, StackOffset). set_registers_args( - #state{used_regs = UsedRegsMask} = State0, + #state{regs = Regs0} = State0, Args, ParamRegs, StackOffset ) -> + UsedRegsMask = jit_regs:used_regs(Regs0), ArgsRegs = args_regs(Args), ParamMask = regs_to_mask(ParamRegs), ArgsMask = regs_to_mask(ArgsRegs), @@ -2169,8 +2139,11 @@ set_registers_args( ), State1#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), - used_regs = ParamMask bor NewUsedMask + regs = jit_regs:set_masks( + State1#state.regs, + ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), + ParamMask bor NewUsedMask + ) }. parameter_regs(Args) -> @@ -2287,11 +2260,12 @@ set_registers_args1( Stream1 = StreamModule:append(Stream0, I), State#state{stream = Stream1}; set_registers_args1( - #state{available_regs = AvailRegs} = State, + #state{regs = Regs} = State, {y_reg, X}, Reg, _StackOffset ) -> + AvailRegs = jit_regs:available_regs(Regs), ldr_y_reg(State, Reg, X, AvailRegs); set_registers_args1( #state{stream_module = StreamModule, stream = Stream0} = State, ArgReg, Reg, _StackOffset @@ -2345,10 +2319,11 @@ move_to_vm_register_emit(State0, Src, {ptr, Reg}) when is_atom(Src) -> Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), State0#state{stream = Stream1}; move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, Src, {y_reg, Y} + #state{regs = Regs0} = State0, Src, {y_reg, Y} ) when is_atom(Src) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp1)), Code = str_y_reg(Src, Y, Temp1, AT), @@ -2363,10 +2338,11 @@ move_to_vm_register_emit( State0#state{stream = Stream1, regs = Regs2}; % Source is an integer to y_reg (optimized: ldr first, then movs) move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, N, {y_reg, Y} + #state{regs = Regs0} = State0, N, {y_reg, Y} ) when is_integer(N), N >= 0, N =< 255 -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail1 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail1), @@ -2383,63 +2359,75 @@ move_to_vm_register_emit( end, State0#state{stream = Stream1, regs = Regs2}; % Source is an integer (0-255 for movs, negative values need different handling) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N), N >= 0, N =< 255 -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_armv6m_asm:movs(Temp, N), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {imm, N}), - State1#state{available_regs = AR0, regs = Regs1}; + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = jit_armv6m_asm:movs(Temp, N), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); %% Handle large values using simple literal pool (branch-over pattern) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, N), - State2 = move_to_vm_register(State1, Temp, Dest), - Regs1 = jit_regs:set_contents(State2#state.regs, Temp, {imm, N}), - State2#state{available_regs = AR0, regs = Regs1}; + with_temp( + State0, + fun(StateT, Temp, _AT) -> + State1 = mov_immediate(StateT, Temp, N), + move_to_vm_register(State1, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); % Source is a VM register -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, extra}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_armv6m_asm:ldr(Temp, ?X_REG(?MAX_REG)), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, ?MAX_REG}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, X}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_armv6m_asm:ldr(Temp, ?X_REG(X)), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, X}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {ptr, Reg}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_armv6m_asm:ldr(Temp, {Reg, 0}), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {y_reg, Y}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - State0a = ldr_y_reg(State0#state{available_regs = AT}, Temp, Y, AT), - State1 = move_to_vm_register(State0a, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {y_reg, Y}), - State1#state{available_regs = AR0, regs = Regs1}; +move_to_vm_register_emit(State0, {x_reg, extra}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = jit_armv6m_asm:ldr(Temp, ?X_REG(?MAX_REG)), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, ?MAX_REG}) end + ); +move_to_vm_register_emit(State0, {x_reg, X}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = jit_armv6m_asm:ldr(Temp, ?X_REG(X)), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, X}) end + ); +move_to_vm_register_emit(State0, {ptr, Reg}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = jit_armv6m_asm:ldr(Temp, {Reg, 0}), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:invalidate_reg(Regs, Temp) end + ); +move_to_vm_register_emit(State0, {y_reg, Y}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, AT) -> + State0a = ldr_y_reg(StateT, Temp, Y, AT), + move_to_vm_register(State0a, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {y_reg, Y}) end + ); % term_to_float move_to_vm_register_emit( #state{ stream_module = StreamModule, - available_regs = Avail, + regs = Regs0, stream = Stream0, variant = Variant } = @@ -2447,6 +2435,7 @@ move_to_vm_register_emit( {free, {ptr, Reg, 1}}, {fp_reg, F} ) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), I1 = jit_armv6m_asm:ldr(Temp1, ?FP_REGS), @@ -2468,6 +2457,19 @@ move_to_vm_register_emit( Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp1), Temp2), State1#state{stream = Stream1, regs = Regs1}. +-spec with_temp( + state(), + fun((state(), armv6m_register(), non_neg_integer()) -> state()), + fun((jit_regs:regs(), armv6m_register()) -> jit_regs:regs()) +) -> state(). +with_temp(#state{regs = Regs0} = State0, EmitFun, ContentsFun) -> + AR0 = jit_regs:available_regs(Regs0), + Temp = first_avail(AR0), + AT = AR0 band (bnot reg_bit(Temp)), + State1 = EmitFun(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, AT), + Regs1 = jit_regs:set_available_regs(ContentsFun(State1#state.regs, Temp), AR0), + State1#state{regs = Regs1}. + %%----------------------------------------------------------------------------- %% @doc Emit a move of an array element (reg[x]) to a vm or a native register. %% @end @@ -2484,12 +2486,13 @@ move_to_vm_register_emit( vm_register() | armv6m_register() ) -> state(). move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) andalso Index * 4 =< 124 -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_armv6m_asm:ldr(Temp, {Reg, Index * 4}), I2 = jit_armv6m_asm:str(Temp, ?X_REG(X)), @@ -2498,12 +2501,13 @@ move_array_element( Regs2 = jit_regs:set_contents(Regs1, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, available_regs = Avail} = + #state{stream_module = StreamModule, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), % For large offsets, use max offset (124) in ldr + remainder in temp register @@ -2525,12 +2529,13 @@ move_array_element( Regs3 = jit_regs:set_contents(Regs2, Temp2, {x_reg, X}), State1#state{stream = Stream2, regs = Regs3}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) andalso Index * 4 =< 124 -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_armv6m_asm:ldr(Temp, {Reg, Index * 4}), I2 = jit_armv6m_asm:str(Temp, {Dest, 0}), @@ -2538,12 +2543,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, available_regs = Avail} = + #state{stream_module = StreamModule, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % For large offsets, use max offset (124) in ldr + remainder in temp register Offset = Index * 4, @@ -2559,12 +2565,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), State1#state{stream = Stream2, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) andalso Index * 4 =< 124 -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail1 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail1), @@ -2579,13 +2586,14 @@ move_array_element( State#state{stream = Stream1, regs = Regs3}; move_array_element( #state{ - stream_module = StreamModule, available_regs = Avail + stream_module = StreamModule, regs = Regs0 } = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail1 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail1), @@ -2606,12 +2614,13 @@ move_array_element( Regs3 = jit_regs:invalidate_reg(Regs2, Temp2), State1#state{stream = Stream2, regs = Regs3}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Index, {y_reg, Y} ) when is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), I1 = jit_armv6m_asm:ldr(Reg, {Reg, Index * 4}), @@ -2633,8 +2642,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2645,14 +2652,13 @@ move_array_element( I2 = jit_armv6m_asm:ldr(IndexReg, {Reg, IndexReg}), I3 = jit_armv6m_asm:str(IndexReg, ?X_REG(X)), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_vm_loc(Regs0, {x_reg, X}), - Regs2 = jit_regs:invalidate_reg(Regs1, IndexReg), + Regs2 = jit_regs:free_reg( + jit_regs:invalidate_reg(Regs1, IndexReg), + Bit + ), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, regs = Regs2 }; @@ -2660,8 +2666,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2672,13 +2676,12 @@ move_array_element( I2 = jit_armv6m_asm:ldr(IndexReg, {Reg, IndexReg}), I3 = jit_armv6m_asm:str(IndexReg, {PtrReg, 0}), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), - Regs1 = jit_regs:invalidate_reg(Regs0, IndexReg), + Regs1 = jit_regs:free_reg( + jit_regs:invalidate_reg(Regs0, IndexReg), + Bit + ), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, regs = Regs1 }; @@ -2686,14 +2689,13 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, {free, IndexReg}, {y_reg, Y} ) when is_atom(IndexReg) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), I1 = jit_armv6m_asm:lsls(IndexReg, IndexReg, 2), I2 = jit_armv6m_asm:ldr(IndexReg, {Reg, IndexReg}), Temp = first_avail(AvailableRegs0), @@ -2702,17 +2704,16 @@ move_array_element( Code = str_y_reg(IndexReg, Y, Temp, AT), I3 = Code, Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append( Stream0, <> ), Regs1 = jit_regs:invalidate_reg(Regs0, IndexReg), Regs2 = jit_regs:invalidate_vm_loc(Regs1, {y_reg, Y}), - Regs3 = jit_regs:invalidate_reg(Regs2, Temp), + Regs3 = jit_regs:free_reg( + jit_regs:invalidate_reg(Regs2, Temp), + Bit + ), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, regs = Regs3 }. @@ -2736,11 +2737,12 @@ get_array_element( get_array_element( #state{ stream_module = StreamModule, - available_regs = Avail + regs = Regs0 } = State, {free, Reg}, Index ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % For large offsets, split into ldr immediate (max 124) + remainder in temp register Offset = Index * 4, @@ -2757,23 +2759,23 @@ get_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, Index ) when Index * 4 =< 124 -> + Avail = jit_regs:available_regs(Regs0), ElemReg = first_avail(Avail), ElemBit = reg_bit(ElemReg), I1 = jit_armv6m_asm:ldr(ElemReg, {Reg, Index * 4}), Stream1 = StreamModule:append(Stream0, <>), - Regs1 = jit_regs:invalidate_reg(Regs0, ElemReg), + Regs1 = jit_regs:alloc_reg( + jit_regs:invalidate_reg(Regs0, ElemReg), + ElemBit + ), { State#state{ stream = Stream1, - available_regs = Avail band (bnot ElemBit), - used_regs = UsedRegs0 bor ElemBit, regs = Regs1 }, ElemReg @@ -2781,12 +2783,12 @@ get_array_element( get_array_element( #state{ stream_module = StreamModule, - available_regs = Avail, - used_regs = UsedRegs0 + regs = Regs0 } = State, Reg, Index ) -> + Avail = jit_regs:available_regs(Regs0), ElemReg = first_avail(Avail), ElemBit = reg_bit(ElemReg), Avail1 = Avail band (bnot ElemBit), @@ -2801,12 +2803,13 @@ get_array_element( I2 = jit_armv6m_asm:ldr(ElemReg, {Temp, 124}), Stream2 = StreamModule:append(Stream1, <>), Regs1 = jit_regs:invalidate_reg(State1#state.regs, ElemReg), - Regs2 = jit_regs:invalidate_reg(Regs1, Temp), + Regs2 = jit_regs:alloc_reg( + jit_regs:invalidate_reg(Regs1, Temp), + ElemBit + ), { State1#state{ stream = Stream2, - available_regs = Avail1, - used_regs = UsedRegs0 bor ElemBit, regs = Regs2 }, ElemReg @@ -2826,11 +2829,12 @@ move_to_array_element( Stream1 = StreamModule:append(Stream0, I1), State0#state{stream = Stream1}; move_to_array_element( - #state{stream_module = StreamModule, available_regs = Avail} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, ValueReg, Reg, Index ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), % For large offsets, split into str immediate (max 124) + remainder in temp register Offset = Index * 4, @@ -2844,12 +2848,13 @@ move_to_array_element( Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), State1#state{stream = Stream2, regs = Regs1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, ValueReg, Reg, IndexReg ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(Reg) andalso ?IS_GPR(IndexReg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_armv6m_asm:mov(Temp, IndexReg), I2 = jit_armv6m_asm:lsls(Temp, Temp, 2), @@ -2876,13 +2881,14 @@ move_to_array_element( ) when is_integer(IndexReg) andalso is_integer(Offset) andalso Offset div 8 =:= 0 -> move_to_array_element(State, Value, BaseReg, IndexReg + (Offset div 8)); move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, ValueReg, BaseReg, IndexReg, Offset ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_armv6m_asm:adds(Temp, IndexReg, Offset), I2 = jit_armv6m_asm:lsls(Temp, Temp, 2), @@ -2898,7 +2904,7 @@ move_to_array_element( Offset ) -> {State1, ValueReg} = copy_to_native_register(State0, Value), - Temp = first_avail(State1#state.available_regs), + Temp = first_avail(jit_regs:available_regs(State1#state.regs)), I1 = jit_armv6m_asm:adds(Temp, IndexReg, Offset), I2 = jit_armv6m_asm:lsls(Temp, Temp, 2), I3 = jit_armv6m_asm:str(ValueReg, {BaseReg, Temp}), @@ -2917,16 +2923,17 @@ move_to_native_register(#state{regs = Regs} = State, Value) -> case Contents =/= unknown andalso jit_regs:find_reg_with_contents(Regs, Contents) of {ok, CachedReg} -> Bit = reg_bit(CachedReg), - case State#state.used_regs band Bit of + Used = jit_regs:used_regs(Regs), + Avail = jit_regs:available_regs(Regs), + case Used band Bit of 0 -> - case State#state.available_regs band Bit of + case Avail band Bit of 0 -> move_to_native_register_emit(State, Value, Contents); _ -> { State#state{ - used_regs = State#state.used_regs bor Bit, - available_regs = State#state.available_regs band (bnot Bit) + regs = jit_regs:alloc_reg(Regs, Bit) }, CachedReg } @@ -2942,13 +2949,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, cp, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_armv6m_asm:ldr(Reg, ?CP), @@ -2957,9 +2963,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2974,8 +2978,6 @@ move_to_native_register_emit( {State#state{stream = Stream1, regs = Regs1}, Reg}; move_to_native_register_emit( #state{ - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State0, Imm, @@ -2983,26 +2985,24 @@ move_to_native_register_emit( ) when is_integer(Imm) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), Regs1 = jit_regs:set_contents(Regs0, Reg, Contents), State1 = State0#state{ - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, {move_to_native_register(State1, Imm, Reg), Reg}; move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, extra}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_armv6m_asm:ldr(Reg, ?X_REG(?MAX_REG)), @@ -3011,9 +3011,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -3021,8 +3019,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, X}, @@ -3030,6 +3026,7 @@ move_to_native_register_emit( ) when X < ?MAX_REG -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_armv6m_asm:ldr(Reg, ?X_REG(X)), @@ -3038,25 +3035,23 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; move_to_native_register_emit( #state{ - available_regs = Avail, - used_regs = Used + regs = Regs0 } = State, {y_reg, Y}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), AvailT = Avail band (bnot Bit), State1 = ldr_y_reg( - State#state{available_regs = AvailT, used_regs = Used bor Bit}, + State#state{regs = jit_regs:alloc_reg(Regs0, Bit)}, Reg, Y, AvailT @@ -3070,25 +3065,27 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used + regs = Regs0 } = State, {fp_reg, F}, _Contents ) -> + Avail = jit_regs:available_regs(Regs0), RegA = first_avail(Avail), BitA = reg_bit(RegA), Avail1 = Avail band (bnot BitA), RegB = first_avail(Avail1), BitB = reg_bit(RegB), - AvailT = Avail1 band (bnot BitB), I1 = jit_armv6m_asm:ldr(RegB, ?FP_REGS), I2 = jit_armv6m_asm:ldr(RegA, {RegB, F * 8}), I3 = jit_armv6m_asm:ldr(RegB, {RegB, F * 8 + 4}), Code = <>, Stream1 = StreamModule:append(Stream0, Code), { - State#state{stream = Stream1, available_regs = AvailT, used_regs = Used bor BitA bor BitB}, + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs0, BitA bor BitB) + }, {fp, RegA, RegB} }. @@ -3132,10 +3129,11 @@ move_to_native_register( Regs1 = jit_regs:set_contents(Regs0, RegDst, {x_reg, X}), State#state{stream = Stream1, regs = Regs1}; move_to_native_register( - #state{available_regs = AT} = State, + #state{regs = Regs} = State, {y_reg, Y}, RegDst ) -> + AT = jit_regs:available_regs(Regs), State1 = ldr_y_reg(State, RegDst, Y, AT), #state{regs = Regs0} = State1, Regs1 = jit_regs:set_contents(Regs0, RegDst, {y_reg, Y}), @@ -3160,12 +3158,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, Reg ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = jit_armv6m_asm:mov(SaveReg, Reg), @@ -3175,9 +3172,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, SaveBit) }, SaveReg }; @@ -3185,12 +3180,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {ptr, Reg} ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = jit_armv6m_asm:ldr(SaveReg, {Reg, 0}), @@ -3199,9 +3193,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, SaveBit) }, SaveReg }; @@ -3209,21 +3201,27 @@ copy_to_native_register(State, Reg) -> move_to_native_register(State, Reg). move_to_cp( - #state{available_regs = Avail} = State, + #state{regs = Regs0} = State, {y_reg, Y} ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), + % The saved cp spans two slots: y[Y] = offset word (-> ?CP), y[Y+1] = Module* + % (-> ?CP_MODULE). Copy both into ctx->cp. State1 = ldr_y_reg(State, Reg, Y, AvailT), - I2 = jit_armv6m_asm:str(Reg, ?CP), - Stream1 = (State1#state.stream_module):append(State1#state.stream, I2), - State1#state{stream = Stream1}. + SM = State1#state.stream_module, + Stream1 = SM:append(State1#state.stream, jit_armv6m_asm:str(Reg, ?CP)), + State2 = ldr_y_reg(State1#state{stream = Stream1}, Reg, Y + 1, AvailT), + Stream2 = SM:append(State2#state.stream, jit_armv6m_asm:str(Reg, ?CP_MODULE)), + State2#state{stream = Stream2}. increment_sp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Offset ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), I1 = jit_armv6m_asm:ldr(Reg, ?Y_REGS), I2 = jit_armv6m_asm:adds(Reg, Offset * 4), @@ -3238,10 +3236,11 @@ set_continuation_to_label( stream_module = StreamModule, stream = Stream0, offset = JumpTableOffset, - available_regs = Avail + regs = Regs0 } = State, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), % Calculate jump table entry offset @@ -3284,11 +3283,11 @@ set_continuation_to_offset( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempJitState = first_avail(Avail band (bnot reg_bit(Temp))), OffsetRef = make_ref(), @@ -3333,11 +3332,10 @@ get_module_index( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), Avail1 = Avail band (bnot RegBit), @@ -3353,13 +3351,53 @@ get_module_index( { State#state{ stream = Stream1, - available_regs = Avail1, - used_regs = UsedRegs0 bor RegBit, - regs = Regs2 + regs = jit_regs:alloc_reg(Regs2, RegBit) }, Reg }. +%% @doc Load the current module pointer (jit_state->module) into a register. +get_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + Avail1 = Avail band (bnot RegBit), + TempJitState = first_avail(Avail1), + I1a = jit_armv6m_asm:ldr(TempJitState, {sp, ?STACK_OFFSET_JITSTATE}), + I1b = jit_armv6m_asm:ldr(Reg, ?JITSTATE_MODULE(TempJitState)), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, TempJitState), Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the Module pointer stored in ctx->cp (?CP_MODULE) into a register. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_armv6m_asm:ldr(Reg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the offset word (offset << 2) stored in ctx->cp (?CP) into a register. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + I = jit_armv6m_asm:ldr(Reg, ?CP), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. %% @doc Perform an AND of a register with an immediate. %% JIT currentl calls this with two values: ?TERM_PRIMARY_CLEAR_MASK (-4) to %% clear bits and ?TERM_BOXED_TAG_MASK (0x3F). We can avoid any literal pool @@ -3386,36 +3424,83 @@ and_( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {State0#state{stream = Stream1, regs = Regs1}, Reg}; and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{regs = Regs0} = State0, {free, Reg}, Val +) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), + and_imm_impl(Avail, State0, Reg, Val); +and_( + #state{stream_module = StreamModule, regs = Regs0} = + State0, + Reg, + ?TERM_PRIMARY_CLEAR_MASK +) -> + Avail = jit_regs:available_regs(Regs0), + ResultReg = first_avail(Avail), + ResultBit = reg_bit(ResultReg), + I1 = jit_armv6m_asm:lsrs(ResultReg, Reg, 2), + I2 = jit_armv6m_asm:lsls(ResultReg, ResultReg, 2), + Stream1 = StreamModule:append(State0#state.stream, <>), + Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), + { + State0#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, ResultBit) + }, + ResultReg + }. + +and_imm_impl( + Avail, + #state{stream_module = StreamModule, regs = Regs0} = State0, + Reg, + Val ) when Avail =/= 0 andalso Val < 0 andalso Val >= -256 -> Temp = first_avail(Avail), TempBit = reg_bit(Temp), AT = Avail band (bnot TempBit), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, bnot (Val)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, bnot (Val) + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:bics(Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = AT bor TempBit, stream = Stream2, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, - {free, Reg}, + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, AT bor TempBit) + }, + Reg + }; +and_imm_impl( + Avail, + #state{stream_module = StreamModule, regs = Regs0} = State0, + Reg, Val ) when Avail =/= 0 -> Temp = first_avail(Avail), TempBit = reg_bit(Temp), AT = Avail band (bnot TempBit), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:ands(Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = AT bor TempBit, stream = Stream2, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = 0} = State0, - {free, Reg}, + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, AT bor TempBit) + }, + Reg + }; +and_imm_impl( + 0, + #state{stream_module = StreamModule} = State0, + Reg, Val ) when Val < 0 andalso Val >= -256 -> % No available registers, use r0 as temp and save it to r12 @@ -3434,9 +3519,10 @@ and_( Stream4 = StreamModule:append(Stream3, Restore), Regs1 = jit_regs:invalidate_reg(State0#state.regs, Reg), {State0#state{stream = Stream4, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = 0, regs = Regs0} = State0, - {free, Reg}, +and_imm_impl( + 0, + #state{stream_module = StreamModule, regs = Regs0} = State0, + Reg, Val ) -> % No available registers, use r0 as temp and save it to r12 @@ -3454,28 +3540,7 @@ and_( Restore = jit_armv6m_asm:mov(r0, ?IP_REG), Stream4 = StreamModule:append(Stream3, Restore), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), - {State0#state{stream = Stream4, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = Avail, used_regs = UR, regs = Regs0} = - State0, - Reg, - ?TERM_PRIMARY_CLEAR_MASK -) -> - ResultReg = first_avail(Avail), - ResultBit = reg_bit(ResultReg), - I1 = jit_armv6m_asm:lsrs(ResultReg, Reg, 2), - I2 = jit_armv6m_asm:lsls(ResultReg, ResultReg, 2), - Stream1 = StreamModule:append(State0#state.stream, <>), - Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), - { - State0#state{ - stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 - }, - ResultReg - }. + {State0#state{stream = Stream4, regs = Regs1}, Reg}. or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Reg, SrcReg) when is_atom(SrcReg) @@ -3485,20 +3550,34 @@ or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; or_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{regs = Regs0} = State0, + Reg, + Val +) -> + or_imm_impl(jit_regs:available_regs(Regs0), State0, Reg, Val). + +or_imm_impl( + Avail, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when Avail =/= 0 -> Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:orrs(Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}; -or_( - #state{stream_module = StreamModule, available_regs = 0, regs = Regs0} = State0, + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }; +or_imm_impl( + 0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> @@ -3525,20 +3604,34 @@ xor_( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; xor_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{regs = Regs0} = State0, + Reg, + Val +) -> + xor_imm_impl(jit_regs:available_regs(Regs0), State0, Reg, Val). + +xor_imm_impl( + Avail, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when Avail =/= 0 -> Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:eors(Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}; -xor_( - #state{stream_module = StreamModule, available_regs = 0, regs = Regs0} = State0, + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }; +xor_imm_impl( + 0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> @@ -3562,15 +3655,21 @@ add(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; -add(#state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, Reg, Val) -> +add(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:adds(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }. mov_immediate(#state{stream_module = StreamModule, stream = Stream0} = State, Reg, Val) when Val >= 0 andalso Val =< 255 @@ -3740,23 +3839,29 @@ sub(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I1), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State#state{stream = Stream1, regs = Regs1}; -sub(#state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, Reg, Val) -> +sub(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:subs(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }. -spec mul(state(), armv6m_register(), integer() | armv6m_register()) -> state(). mul(State, _Reg, 1) -> State; mul(State, Reg, 2) -> shift_left(State, Reg, 1); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> - Temp = first_avail(Avail), +mul(#state{regs = Regs0} = State, Reg, 3) -> + Temp = first_avail(jit_regs:available_regs(Regs0)), I1 = jit_armv6m_asm:lsls(Temp, Reg, 1), I2 = jit_armv6m_asm:adds(Reg, Temp, Reg), Stream1 = (State#state.stream_module):append(State#state.stream, <>), @@ -3764,8 +3869,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 4) -> shift_left(State, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> - Temp = first_avail(Avail), +mul(#state{regs = Regs0} = State, Reg, 5) -> + Temp = first_avail(jit_regs:available_regs(Regs0)), I1 = jit_armv6m_asm:lsls(Temp, Reg, 2), I2 = jit_armv6m_asm:adds(Reg, Temp, Reg), Stream1 = (State#state.stream_module):append(State#state.stream, <>), @@ -3774,8 +3879,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> mul(State0, Reg, 6) -> State1 = mul(State0, Reg, 3), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> - Temp = first_avail(Avail), +mul(#state{regs = Regs0} = State, Reg, 7) -> + Temp = first_avail(jit_regs:available_regs(Regs0)), I1 = jit_armv6m_asm:lsls(Temp, Reg, 3), I2 = jit_armv6m_asm:subs(Reg, Temp, Reg), Stream1 = (State#state.stream_module):append(State#state.stream, <>), @@ -3783,8 +3888,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 8) -> shift_left(State, Reg, 3); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> - Temp = first_avail(Avail), +mul(#state{regs = Regs0} = State, Reg, 9) -> + Temp = first_avail(jit_regs:available_regs(Regs0)), I1 = jit_armv6m_asm:lsls(Temp, Reg, 3), I2 = jit_armv6m_asm:adds(Reg, Temp, Reg), Stream1 = (State#state.stream_module):append(State#state.stream, <>), @@ -3793,8 +3898,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> mul(State0, Reg, 10) -> State1 = mul(State0, Reg, 5), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 15) -> - Temp = first_avail(Avail), +mul(#state{regs = Regs0} = State, Reg, 15) -> + Temp = first_avail(jit_regs:available_regs(Regs0)), I1 = jit_armv6m_asm:lsls(Temp, Reg, 4), I2 = jit_armv6m_asm:subs(Reg, Temp, Reg), Stream1 = (State#state.stream_module):append(State#state.stream, <>), @@ -3807,20 +3912,25 @@ mul(State, Reg, 32) -> mul(State, Reg, 64) -> shift_left(State, Reg, 6); mul( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempBit = reg_bit(Temp), AT = Avail band (bnot TempBit), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_armv6m_asm:muls(Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Temp), Reg), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp), Reg), + State1Avail = jit_regs:available_regs(State1#state.regs), State1#state{ - stream = Stream2, available_regs = State1#state.available_regs bor TempBit, regs = Regs1 + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, State1Avail bor TempBit) }; mul( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DestReg, SrcReg @@ -3853,9 +3963,10 @@ mul( -spec decrement_reductions_and_maybe_schedule_next(state()) -> state(). decrement_reductions_and_maybe_schedule_next( #state{ - stream_module = StreamModule, stream = Stream0, available_regs = Avail + stream_module = StreamModule, stream = Stream0, regs = Regs0 } = State0 ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempJitState = first_avail(Avail band (bnot reg_bit(Temp))), % Load jit_state pointer from stack @@ -3911,9 +4022,8 @@ decrement_reductions_and_maybe_schedule_next( Stream5 = StreamModule:replace( Stream4, BNEOffset, <> ), - State3 = merge_used_regs(State2#state{stream = Stream5}, State1#state.used_regs), %% schedule_next clobbers caller-saved regs; invalidate cache at continuation. - State3#state{regs = jit_regs:invalidate_all(State1#state.regs)}. + State2#state{stream = Stream5, regs = jit_regs:invalidate_all(State1#state.regs)}. -spec call_or_schedule_next(state(), non_neg_integer()) -> state(). call_or_schedule_next(State0, Label) -> @@ -3925,10 +4035,11 @@ call_only_or_schedule_next( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), TempJitState = first_avail(Avail band (bnot reg_bit(Temp))), % Load jit_state pointer from stack @@ -4002,29 +4113,24 @@ call_primitive_with_cp(State0, Primitive, Args) -> -spec set_cp(state()) -> {state(), non_neg_integer(), armv6m_register()}. set_cp(State0) -> - % get module index (dynamically) - { - #state{stream_module = StreamModule, stream = Stream0, available_regs = AvailRegs} = State1, - Reg - } = get_module_index( - State0 - ), - % Get a temporary register from available registers + % cp is two words: store the Module pointer (jit_state->module) at ?CP_MODULE, + % and the return offset << 2 at ?CP (patched by rewrite_cp_offset below). + {#state{stream_module = StreamModule, stream = Stream0} = State1, ModReg} = + get_module(State0), + IModStore = jit_armv6m_asm:str(ModReg, ?CP_MODULE), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + AvailRegs = jit_regs:available_regs(State2#state.regs), + % Get a temporary register to hold the offset value TempReg = first_avail(AvailRegs), - - Offset = StreamModule:offset(Stream0), - % build cp with module_index << 24 - I1 = jit_armv6m_asm:lsls(Reg, Reg, 24), - % Placeholder for offset load instruction - I2 = <<16#FFFF:16>>, - MOVOffset = Offset + byte_size(I1), - % OR the module index with the offset (loaded in temp register) - I3 = jit_armv6m_asm:orrs(Reg, TempReg), - I4 = jit_armv6m_asm:str(Reg, ?CP), - Code = <>, - Stream1 = StreamModule:append(Stream0, Code), - State2 = State1#state{stream = Stream1}, - State3 = free_native_register(State2, Reg), + Offset = StreamModule:offset(Stream1), + % Placeholder for the offset load instruction (patched by rewrite_cp_offset) + I1 = <<16#FFFF:16>>, + MOVOffset = Offset, + I2 = jit_armv6m_asm:str(TempReg, ?CP), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, {State3, MOVOffset, TempReg}. -spec rewrite_cp_offset(state(), non_neg_integer(), armv6m_register()) -> state(). @@ -4075,16 +4181,26 @@ rewrite_cp_offset( State0#state{stream = Stream3}. set_bs( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, TermReg ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), + Temp2 = first_avail(Avail band (bnot reg_bit(Temp))), I1 = jit_armv6m_asm:str(TermReg, ?BS), I2 = jit_armv6m_asm:movs(Temp, 0), - I3 = jit_armv6m_asm:str(Temp, ?BS_OFFSET), - Stream1 = StreamModule:append(Stream0, <>), - Regs1 = jit_regs:invalidate_reg(Regs0, Temp), + % Since cp grew to two words, ?BS_OFFSET sits at 0x80, beyond the Thumb + % str immediate range (max 124). Materialize the address in Temp2 = ctx + + % 0x80, then store with a zero offset. + {?CTX_REG, BsOffsetOff} = ?BS_OFFSET, + I3 = jit_armv6m_asm:movs(Temp2, BsOffsetOff), + I4 = jit_armv6m_asm:add(Temp2, ?CTX_REG), + I5 = jit_armv6m_asm:str(Temp, {Temp2, 0}), + Stream1 = StreamModule:append( + Stream0, <> + ), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Temp), Temp2), State0#state{stream = Stream1, regs = Regs1}. %%----------------------------------------------------------------------------- @@ -4241,57 +4357,6 @@ regs_to_mask([offset | T]) -> regs_to_mask(T); regs_to_mask([stack | T]) -> regs_to_mask(T); regs_to_mask([Reg | T]) -> reg_bit(Reg) bor regs_to_mask(T). -%% first_avail returns the first available register from a bitmask. -%% Order matches AVAILABLE_REGS = [r7, r6, r5, r4, r3, r1] -first_avail(Mask) when Mask band ?REG_BIT_R7 =/= 0 -> r7; -first_avail(Mask) when Mask band ?REG_BIT_R6 =/= 0 -> r6; -first_avail(Mask) when Mask band ?REG_BIT_R5 =/= 0 -> r5; -first_avail(Mask) when Mask band ?REG_BIT_R4 =/= 0 -> r4; -first_avail(Mask) when Mask band ?REG_BIT_R3 =/= 0 -> r3; -first_avail(Mask) when Mask band ?REG_BIT_R1 =/= 0 -> r1. - -%% Convert bitmask to list, matching the order of AVAILABLE_REGS. -mask_to_list(0) -> []; -mask_to_list(Mask) -> mask_to_list_r7(Mask). - -mask_to_list_r7(Mask) when Mask band ?REG_BIT_R7 =/= 0 -> [r7 | mask_to_list_r6(Mask)]; -mask_to_list_r7(Mask) -> mask_to_list_r6(Mask). -mask_to_list_r6(Mask) when Mask band ?REG_BIT_R6 =/= 0 -> [r6 | mask_to_list_r5(Mask)]; -mask_to_list_r6(Mask) -> mask_to_list_r5(Mask). -mask_to_list_r5(Mask) when Mask band ?REG_BIT_R5 =/= 0 -> [r5 | mask_to_list_r4(Mask)]; -mask_to_list_r5(Mask) -> mask_to_list_r4(Mask). -mask_to_list_r4(Mask) when Mask band ?REG_BIT_R4 =/= 0 -> [r4 | mask_to_list_r3(Mask)]; -mask_to_list_r4(Mask) -> mask_to_list_r3(Mask). -mask_to_list_r3(Mask) when Mask band ?REG_BIT_R3 =/= 0 -> [r3 | mask_to_list_r1(Mask)]; -mask_to_list_r3(Mask) -> mask_to_list_r1(Mask). -mask_to_list_r1(Mask) when Mask band ?REG_BIT_R1 =/= 0 -> [r1]; -mask_to_list_r1(_Mask) -> []. - -args_regs(Args) -> - lists:map( - fun - ({free, {ptr, Reg}}) -> Reg; - ({free, Reg}) when is_atom(Reg) -> Reg; - ({free, Imm}) when is_integer(Imm) -> imm; - (offset) -> imm; - (ctx) -> ?CTX_REG; - (jit_state) -> jit_state; - (jit_state_tail_call) -> jit_state; - (stack) -> stack; - (Reg) when is_atom(Reg) -> Reg; - (Imm) when is_integer(Imm) -> imm; - ({ptr, Reg}) -> Reg; - ({x_reg, _}) -> ?CTX_REG; - ({y_reg, _}) -> ?CTX_REG; - ({fp_reg, _}) -> ?CTX_REG; - ({free, {x_reg, _}}) -> ?CTX_REG; - ({free, {y_reg, _}}) -> ?CTX_REG; - ({free, {fp_reg, _}}) -> ?CTX_REG; - ({avm_int64_t, _}) -> imm - end, - Args - ). - %%----------------------------------------------------------------------------- %% @doc Add a label at the current offset. Eventually align it with a nop. %% @end diff --git a/libs/jit/src/jit_backend_regs_impl.hrl b/libs/jit/src/jit_backend_regs_impl.hrl new file mode 100644 index 0000000000..6d515ce8e1 --- /dev/null +++ b/libs/jit/src/jit_backend_regs_impl.hrl @@ -0,0 +1,139 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +%% Shared native-register allocation bookkeeping for the register-based JIT +%% backends. The available/used scratch-register bitmasks live inside +%% `jit_regs:regs()' (set via `jit_regs:new/2', read via `jit_regs:available_regs/1' +%% and `jit_regs:used_regs/1'); these functions are the thin per-backend +%% wrappers that read/write the state record's `regs' field. +%% +%% Backends including this file must provide: +%% - `#state{}' record with a `regs :: jit_regs:regs()' field (no separate +%% `available_regs' / `used_regs' fields — they're inside `regs'), +%% - `reg_bit/1' : map a register atom to its bitmask, +%% - `?AVAILABLE_REGS_MASK' : the mask of all allocatable scratch registers, +%% - `?PARAMETER_REGS' : the C-ABI parameter registers, in order, +%% - `?FIRST_AVAIL_REGS' : allocatable scratch registers in allocation-priority +%% order (drives `first_avail/1'), +%% - `?MASK_TO_LIST_REGS' : every register that can appear in a used/available +%% mask, in canonical order (drives `mask_to_list/1'), +%% - `?CTX_REG' : the register holding the execution context, +%% - `?JITSTATE_ARG_REG' : what the `jit_state' argument maps to (a register +%% atom on x86_64/aarch64, the atom `jit_state' on backends that keep it +%% symbolic). +%% +%% These macros must be defined *before* this file is included. +%% +%% wasm32 does NOT include this file: it allocates dynamically-numbered WASM +%% locals, so its bit helpers use `local_bit/1' + `mask_to_locals/1' rather than +%% the register variants, and its scratch pool grows past the initial +%% `?AVAILABLE_REGS_MASK' at runtime. It defines these bookkeeping functions +%% inline. + +%% Return the list of currently used native registers (debugging only). +used_regs(#state{regs = Regs}) -> mask_to_list(jit_regs:used_regs(Regs)). + +%% Return the list of currently available native scratch registers (debugging +%% only). +available_regs(#state{regs = Regs}) -> mask_to_list(jit_regs:available_regs(Regs)). + +%% Free a list of native registers. Entries that are not registers (pointers to +%% registers, immediates, etc.) are handled/ignored by free_native_register/2. +free_native_registers(State, []) -> + State; +free_native_registers(State, [Reg | Rest]) -> + State1 = free_native_register(State, Reg), + free_native_registers(State1, Rest). + +%% Free a single native register. A bare register atom is returned to the +%% available pool; a {ptr, Reg} frees the underlying register; anything else is +%% ignored. +free_native_register(#state{regs = Regs} = State, Reg) when is_atom(Reg) -> + State#state{regs = jit_regs:free_reg(Regs, reg_bit(Reg))}; +free_native_register(State, {ptr, Reg}) -> + free_native_register(State, Reg); +free_native_register(State, _Other) -> + State. + +%% Assert that all native scratch registers are available (debugging only). +assert_all_native_free(#state{regs = Regs}) -> + 0 = jit_regs:used_regs(Regs), + ?AVAILABLE_REGS_MASK = jit_regs:available_regs(Regs), + ok. + +%% Return the first allocatable scratch register set in `Mask', following the +%% backend's allocation-priority order. Crashes if `Mask' has no allocatable +%% register set; callers check availability beforehand. +first_avail(Mask) -> jit_regs:first_set(Mask, ?FIRST_AVAIL_REGS, fun reg_bit/1). + +%% Return the registers set in `Mask' in the backend's canonical order. The +%% order is significant: callers use this to build the saved-register push/pop +%% sequences around C calls and the argument-register lists, so it directly +%% affects the emitted machine code. +mask_to_list(Mask) -> jit_regs:mask_to_reg_list(Mask, ?MASK_TO_LIST_REGS, fun reg_bit/1). + +%% Map a list of call arguments to the native registers (or `imm'/`stack') that +%% hold them, so the caller can compute which registers a call clobbers. +args_regs(Args) -> lists:map(fun arg_reg/1, Args). + +arg_reg({free, {ptr, Reg}}) -> Reg; +arg_reg({free, Reg}) when is_atom(Reg) -> Reg; +arg_reg({free, Imm}) when is_integer(Imm) -> imm; +arg_reg(offset) -> imm; +arg_reg(ctx) -> ?CTX_REG; +arg_reg(jit_state) -> ?JITSTATE_ARG_REG; +arg_reg(jit_state_tail_call) -> ?JITSTATE_ARG_REG; +arg_reg(stack) -> stack; +arg_reg(Reg) when is_atom(Reg) -> Reg; +arg_reg(Imm) when is_integer(Imm) -> imm; +arg_reg({ptr, Reg}) -> Reg; +arg_reg({x_reg, _}) -> ?CTX_REG; +arg_reg({y_reg, _}) -> ?CTX_REG; +arg_reg({fp_reg, _}) -> ?CTX_REG; +arg_reg({free, {x_reg, _}}) -> ?CTX_REG; +arg_reg({free, {y_reg, _}}) -> ?CTX_REG; +arg_reg({free, {fp_reg, _}}) -> ?CTX_REG; +arg_reg({avm_int64_t, _}) -> imm. + +%% Reserve a single scratch register for a non-returning (tail) call that passes +%% `Args'. Since the frame is not reused, every scratch register except those +%% holding parameters/arguments is free. Returns a map with the chosen scratch +%% register (`temp'), the resulting available/used masks with that register +%% removed from the available pool (`available_mask' / `used_mask'), and the +%% intermediate parameter/argument register lists and masks (some backends thread +%% these into their argument-setup code). +prepare_call_scratch(Args) -> + ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), + ArgsRegs = args_regs(Args), + ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), + ArgsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), + ScratchMask = ?AVAILABLE_REGS_MASK band (bnot (ArgsMask bor ParamMask)), + Temp = first_avail(ScratchMask), + AvailableMask = ScratchMask band (bnot reg_bit(Temp)), + UsedMask = ?AVAILABLE_REGS_MASK band (bnot AvailableMask), + #{ + temp => Temp, + available_mask => AvailableMask, + used_mask => UsedMask, + param_regs => ParamRegs, + args_regs => ArgsRegs, + param_mask => ParamMask, + args_mask => ArgsMask + }. diff --git a/libs/jit/src/jit_regs.erl b/libs/jit/src/jit_regs.erl index 881ceb2ae2..1fff453441 100644 --- a/libs/jit/src/jit_regs.erl +++ b/libs/jit/src/jit_regs.erl @@ -36,6 +36,13 @@ -export([ new/0, + new/2, + available_regs/1, + used_regs/1, + set_available_regs/2, + set_masks/3, + alloc_reg/2, + free_reg/2, get_contents/2, get_all_contents/1, set_contents/3, @@ -46,13 +53,16 @@ invalidate_vm_loc/2, find_reg_with_contents/2, merge/2, + merge/3, stack_push/2, stack_pop/1, stack_clear/1, stack_contents/1, value_to_contents/2, vm_dest_to_contents/2, - regs_to_mask/2 + regs_to_mask/2, + first_set/3, + mask_to_reg_list/3 ]). -export_type([regs/0, contents/0]). @@ -77,7 +87,9 @@ -record(regs, { contents = #{} :: #{atom() => contents()}, stack = [] :: [atom() | contents()], - unreachable = false :: boolean() + unreachable = false :: boolean(), + available_regs = 0 :: non_neg_integer(), + used_regs = 0 :: non_neg_integer() }). -opaque regs() :: #regs{}. @@ -87,6 +99,52 @@ new() -> #regs{}. +%% @doc Create a new register tracking state with initial available/used masks. +-spec new(non_neg_integer(), non_neg_integer()) -> regs(). +new(Available, Used) -> + #regs{available_regs = Available, used_regs = Used}. + +%% @doc Get the available-registers bitmask. +-spec available_regs(regs()) -> non_neg_integer(). +available_regs(#regs{available_regs = A}) -> A. + +%% @doc Get the used-registers bitmask. +-spec used_regs(regs()) -> non_neg_integer(). +used_regs(#regs{used_regs = U}) -> U. + +%% @doc Set the available-registers bitmask. +-spec set_available_regs(regs(), non_neg_integer()) -> regs(). +set_available_regs(#regs{} = Regs, A) -> + Regs#regs{available_regs = A}. + +%% @doc Set both available and used bitmasks at once. +-spec set_masks(regs(), non_neg_integer(), non_neg_integer()) -> regs(). +set_masks(#regs{} = Regs, A, U) -> + Regs#regs{available_regs = A, used_regs = U}. + +%% @doc Mark the scratch register(s) in `Bit' as allocated: remove them from the +%% available pool and add them to the used set. `Bit' is usually a single +%% register's bit (`reg_bit(Reg)') but may be the bitwise-or of several. This is +%% the single place the allocation bit-law lives. +-spec alloc_reg(regs(), non_neg_integer()) -> regs(). +alloc_reg(#regs{available_regs = A, used_regs = U} = Regs, Bit) -> + Regs#regs{available_regs = A band (bnot Bit), used_regs = U bor Bit}. + +%% @doc Mark the scratch register(s) in `Bit' as freed: return them to the +%% available pool and remove them from the used set. Inverse of alloc_reg/2. +-spec free_reg(regs(), non_neg_integer()) -> regs(). +free_reg(#regs{available_regs = A, used_regs = U} = Regs, Bit) -> + Regs#regs{available_regs = A bor Bit, used_regs = U band (bnot Bit)}. + +%% @doc Merge two regs taking the intersection of available masks and union +%% of used masks (constrained by AllRegsMask). +-spec merge(regs(), regs(), non_neg_integer()) -> regs(). +merge(R1, R2, AllRegsMask) -> + Merged0 = merge(R1, R2), + A1 = (R1#regs.available_regs band R2#regs.available_regs) band AllRegsMask, + U1 = (R1#regs.used_regs bor R2#regs.used_regs) band AllRegsMask, + Merged0#regs{available_regs = A1, used_regs = U1}. + %% @doc Get what a CPU register currently holds. -spec get_contents(regs(), atom()) -> contents(). get_contents(#regs{contents = C}, Reg) -> @@ -213,3 +271,27 @@ regs_to_mask([imm | T], RegBitFn) -> regs_to_mask(T, RegBitFn); regs_to_mask([jit_state | T], RegBitFn) -> regs_to_mask(T, RegBitFn); regs_to_mask([stack | T], RegBitFn) -> regs_to_mask(T, RegBitFn); regs_to_mask([Reg | T], RegBitFn) -> RegBitFn(Reg) bor regs_to_mask(T, RegBitFn). + +%% @doc Return the first register in the ordered register list whose bit (per +%% `RegBitFn') is set in `Mask', following the list order. Crashes with +%% `function_clause' if no register in the list matches — callers must ensure at +%% least one register in the list has its bit set in `Mask' (matching the +%% previous per-backend guard-clause `first_avail/1'). +-spec first_set(non_neg_integer(), [atom()], fun((atom()) -> non_neg_integer())) -> atom(). +first_set(Mask, [Reg | Rest], RegBitFn) -> + case Mask band RegBitFn(Reg) of + 0 -> first_set(Mask, Rest, RegBitFn); + _ -> Reg + end. + +%% @doc Return the registers in the ordered register list whose bits (per +%% `RegBitFn') are set in `Mask', preserving the list order. +-spec mask_to_reg_list(non_neg_integer(), [atom()], fun((atom()) -> non_neg_integer())) -> + [atom()]. +mask_to_reg_list(_Mask, [], _RegBitFn) -> + []; +mask_to_reg_list(Mask, [Reg | Rest], RegBitFn) -> + case Mask band RegBitFn(Reg) of + 0 -> mask_to_reg_list(Mask, Rest, RegBitFn); + _ -> [Reg | mask_to_reg_list(Mask, Rest, RegBitFn)] + end. diff --git a/libs/jit/src/jit_riscv32.erl b/libs/jit/src/jit_riscv32.erl index 3a3fd2687a..6ef563e165 100644 --- a/libs/jit/src/jit_riscv32.erl +++ b/libs/jit/src/jit_riscv32.erl @@ -59,6 +59,9 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -173,8 +176,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), tuple()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), regs :: jit_regs:regs() @@ -184,6 +185,7 @@ -type immediate() :: non_neg_integer(). -type vm_register() :: {x_reg, non_neg_integer()} | {y_reg, non_neg_integer()} | {ptr, riscv32_register()}. +-type native_register() :: riscv32_register(). -type value() :: immediate() | vm_register() | riscv32_register() | {ptr, riscv32_register()}. -type arg() :: ctx | jit_state | offset | value() | {free, value()} | {avm_int64_t, integer()}. @@ -211,9 +213,10 @@ -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). -define(JITSTATE_REG, a1). -define(RA_REG, ra). -define(JITSTATE_MODULE_OFFSET, 0). @@ -282,6 +285,13 @@ -include("jit_riscv_impl.hrl"). +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1) +%% is shared across the register-based backends and flows through jit_regs. +%% Must be included after jit_riscv_impl.hrl, on whose register primitives it +%% depends. +-include("jit_backend_regs_impl.hrl"). + -spec word_size() -> 4 | 8. word_size() -> 4. @@ -305,6 +315,61 @@ rem_( Regs1 = jit_regs:invalidate_reg(Regs0, DividendReg), {State#state{stream = Stream1, regs = Regs1}, DividendReg}. +%% @doc Load the current module pointer (jit_state->module) into a fresh +%% register. This is the Module* itself (unlike get_module_index/1 which then +%% dereferences module->module_index). Used by set_cp to store the Module* +%% half of the two-word (32-bit) cp, and by the 32-bit OP_RETURN fast path. +-spec get_module(state()) -> {state(), riscv32_register()}. +get_module( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + %% Reg = jit_state->module (jit_state is in a1) + I = ?LOAD_WORD(Reg, ?JITSTATE_REG, ?JITSTATE_MODULE_OFFSET), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the Module* half of the saved cp (ctx->cp high word, ?CP_MODULE) +%% into a fresh register. 32-bit only: cp spans two words. +-spec get_cp_module(state()) -> {state(), riscv32_register()}. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP_MODULE, + I = ?LOAD_WORD(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + +%% @doc Load the offset half of the saved cp (ctx->cp low word, ?CP) into a +%% fresh register. The value is offset << 2 (TERM_PRIMARY_CP tag); the caller +%% shifts it right by 2. 32-bit only. +-spec get_cp_offset(state()) -> {state(), riscv32_register()}. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP, + I = ?LOAD_WORD(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + Regs2 = jit_regs:alloc_reg(Regs1, RegBit), + {State#state{stream = Stream1, regs = Regs2}, Reg}. + % ILP32: 64-bit arguments require double-word alignment (even register number) parameter_regs0_avm_int64_t(T, [a0, a1 | Rest], Acc) -> parameter_regs0(T, Rest, [a1, a0 | Acc]); diff --git a/libs/jit/src/jit_riscv64.erl b/libs/jit/src/jit_riscv64.erl index 3dda3760c0..4ed6c8eb0d 100644 --- a/libs/jit/src/jit_riscv64.erl +++ b/libs/jit/src/jit_riscv64.erl @@ -176,11 +176,9 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), tuple()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), - %% Register value tracking for optimization + %% Register value tracking and native-register mask bookkeeping regs :: jit_regs:regs() }). @@ -191,6 +189,7 @@ | {y_reg, non_neg_integer()} | {fp_reg, non_neg_integer()} | {ptr, riscv64_register()}. +-type native_register() :: riscv64_register(). -type value() :: immediate() | vm_register() | riscv64_register() | {ptr, riscv64_register()}. -type arg() :: ctx | jit_state | offset | value() | {free, value()} | {avm_int64_t, integer()}. @@ -297,6 +296,13 @@ -include("jit_riscv_impl.hrl"). +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1) +%% is shared across the register-based backends and flows through jit_regs. +%% Must be included after jit_riscv_impl.hrl, on whose register primitives it +%% depends. +-include("jit_backend_regs_impl.hrl"). + -spec word_size() -> 4 | 8. word_size() -> 8. diff --git a/libs/jit/src/jit_riscv_impl.hrl b/libs/jit/src/jit_riscv_impl.hrl index 389dd6f5ff..5d0f2ebc54 100644 --- a/libs/jit/src/jit_riscv_impl.hrl +++ b/libs/jit/src/jit_riscv_impl.hrl @@ -46,11 +46,9 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, labels = #{}, variant = Variant, - regs = jit_regs:new() + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0) }. %%----------------------------------------------------------------------------- @@ -92,62 +90,10 @@ debugger(#state{stream_module = StreamModule, stream = Stream0} = State) -> Stream1 = StreamModule:append(Stream0, ?ASM:c_ebreak()), State#state{stream = Stream1}. -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently used native registers. This is used for -%% debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of used registers -%%----------------------------------------------------------------------------- -used_regs(#state{used_regs = Used}) -> mask_to_list(Used). - -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently available native scratch registers. This -%% is used for debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of available registers -%%----------------------------------------------------------------------------- -available_regs(#state{available_regs = Available}) -> mask_to_list(Available). - -%%----------------------------------------------------------------------------- -%% @doc Free native registers. The passed list of registers can contain -%% registers, pointer to registers or other values that are ignored. -%% @end -%% @param State current backend state -%% @param Regs list of registers or other values -%% @return The updated backend state -%%----------------------------------------------------------------------------- -free_native_registers(State, []) -> - State; -free_native_registers(State, [Reg | Rest]) -> - State1 = free_native_register(State, Reg), - free_native_registers(State1, Rest). - -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Reg -) when is_atom(Reg) -> - Bit = reg_bit(Reg), - State#state{ - available_regs = Available0 bor Bit, used_regs = Used0 band (bnot Bit) - }; -free_native_register(State, {ptr, Reg}) -> - free_native_register(State, Reg); -free_native_register(State, _Other) -> - State. - -%%----------------------------------------------------------------------------- -%% @doc Assert that all native scratch registers are available. This is used -%% for debugging and not in production. -%% @end -%% @param State current backend state -%% @return ok -%%----------------------------------------------------------------------------- -assert_all_native_free(State) -> - 0 = State#state.used_regs, - ?AVAILABLE_REGS_MASK = State#state.available_regs, - ok. +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1) +%% is provided by the includer (jit_riscv32 / jit_riscv64) via +%% -include("jit_backend_regs_impl.hrl"). %%----------------------------------------------------------------------------- %% @doc Emit the jump table at the beginning of the module. Branches will be @@ -337,30 +283,28 @@ call_primitive( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used + regs = Regs0 } = State, Primitive, Args -) when Available =/= 0 -> - TempReg = first_avail(Available), - TempBit = reg_bit(TempReg), - % Load primitive function pointer - PrepCall = load_primitive_ptr(Primitive, TempReg), - Stream1 = StreamModule:append(Stream0, PrepCall), - StateCall = State#state{ - stream = Stream1, - available_regs = Available band (bnot TempBit), - used_regs = Used bor TempBit, - regs = jit_regs:invalidate_reg(State#state.regs, TempReg) - }, - call_func_ptr(StateCall, {free, TempReg}, Args); -call_primitive( - #state{available_regs = 0} = State, - Primitive, - Args ) -> - call_func_ptr(State, {primitive, Primitive}, Args). + Available = jit_regs:available_regs(Regs0), + case Available of + 0 -> + call_func_ptr(State, {primitive, Primitive}, Args); + _ -> + TempReg = first_avail(Available), + TempBit = reg_bit(TempReg), + % Load primitive function pointer + PrepCall = load_primitive_ptr(Primitive, TempReg), + Stream1 = StreamModule:append(Stream0, PrepCall), + Regs1 = jit_regs:invalidate_reg(Regs0, TempReg), + StateCall = State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, TempBit) + }, + call_func_ptr(StateCall, {free, TempReg}, Args) + end. %%----------------------------------------------------------------------------- %% @doc Emit a jump (call without return) to a primitive with arguments. This @@ -383,23 +327,16 @@ call_primitive_last( % We need a register for the function pointer that should not be used as a parameter % Since we're not returning, we can use all scratch registers except % registers used for parameters - ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), - ArgsRegs = args_regs(Args), - ArgsRegsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), - ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), - ScratchMask = ?AVAILABLE_REGS_MASK band (bnot (ArgsRegsMask bor ParamMask)), - Temp = first_avail(ScratchMask), - TempBit = reg_bit(Temp), - AvailableRegs1 = ScratchMask band (bnot TempBit), - UsedMask = ?AVAILABLE_REGS_MASK band (bnot AvailableRegs1), + #{temp := Temp, available_mask := AvailableRegs1, used_mask := UsedMask} = + prepare_call_scratch(Args), PrepCall = load_primitive_ptr(Primitive, Temp), Stream1 = StreamModule:append(Stream0, PrepCall), State1 = State0#state{ stream = Stream1, - available_regs = AvailableRegs1, - used_regs = UsedMask, - regs = jit_regs:invalidate_reg(State0#state.regs, Temp) + regs = jit_regs:set_masks( + jit_regs:invalidate_reg(State0#state.regs, Temp), AvailableRegs1, UsedMask + ) }, % Preprocess offset special arg @@ -424,9 +361,9 @@ call_primitive_last( tail_call_with_jit_state_registers_only(State2, Temp) end, State4#state{ - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State4#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State4#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) }. %%----------------------------------------------------------------------------- @@ -463,8 +400,7 @@ return_if_not_equal_to_ctx( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State, {free, Reg} ) -> @@ -484,9 +420,7 @@ return_if_not_equal_to_ctx( RegBit = reg_bit(Reg), State#state{ stream = Stream1, - available_regs = AvailableRegs0 bor RegBit, - used_regs = UsedRegs0 band (bnot RegBit), - regs = State#state.regs + regs = jit_regs:free_reg(Regs0, RegBit) }. %%----------------------------------------------------------------------------- @@ -529,11 +463,12 @@ jump_to_continuation( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, + regs = Regs0, offset = BaseOffset } = State0, {free, OffsetReg} ) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), % Calculate absolute address: native_code_base + target_offset % where native_code_base = current_pc + (BaseOffset - CurrentStreamOffset) @@ -552,9 +487,9 @@ jump_to_continuation( % Free all registers since this is a tail jump State0#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State0#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(Regs0), ?AVAILABLE_REGS_MASK, 0 + ) }. branch_to_offset_code(_State, Offset, TargetOffset) when @@ -564,8 +499,10 @@ branch_to_offset_code(_State, Offset, TargetOffset) when Rel = TargetOffset - Offset, ?ASM:j(Rel); branch_to_offset_code( - #state{available_regs = Available}, Offset, TargetOffset -) when Available =/= 0 -> + #state{regs = Regs0}, Offset, TargetOffset +) -> + Available = jit_regs:available_regs(Regs0), + true = Available =/= 0, TempReg = first_avail(Available), % Far branch: use auipc + jalr sequence for PC-relative addressing % This computes: PC + Immediate and jumps to it @@ -594,10 +531,16 @@ branch_to_label_code(State, Offset, Label, {Label, LabelOffset}) -> CodeBlock = branch_to_offset_code(State, Offset, LabelOffset), {State, CodeBlock}; branch_to_label_code( - #state{available_regs = Available, branches = Branches} = State0, Offset, Label, false -) when Available =/= 0 -> - TempReg = first_avail(Available), + #state{regs = Regs0, branches = Branches} = State0, Offset, Label, false +) -> + Available = jit_regs:available_regs(Regs0), + TempReg = + case Available of + 0 -> t6; + _ -> first_avail(Available) + end, % RISC-V: Far branch sequence using PC-relative auipc + jalr (8 bytes) + % When no scratch register is available, use t6 (caller-saved, safe to clobber). % Placeholder: auipc TempReg, 0 % Placeholder: jalr zero, TempReg, 0 @@ -606,20 +549,7 @@ branch_to_label_code( BrEntry = {Offset, {far_branch, TempReg}}, State1 = State0#state{branches = Branches#{Label => [BrEntry | maps:get(Label, Branches, [])]}}, {State1, CodeBlock}; -branch_to_label_code( - #state{available_regs = 0, branches = Branches} = State0, Offset, Label, false -) -> - % RISC-V: Use t6 as scratch (caller-saved, safe to clobber) - % Far branch sequence using PC-relative auipc + jalr (8 bytes) - - % Placeholder: auipc t6, 0 - % Placeholder: jalr zero, t6, 0 - CodeBlock = <<16#FFFFFFFF:32, 16#FFFFFFFF:32>>, - % Add relocation entry - BrEntry = {Offset, {far_branch, t6}}, - State1 = State0#state{branches = Branches#{Label => [BrEntry | maps:get(Label, Branches, [])]}}, - {State1, CodeBlock}; -branch_to_label_code(#state{available_regs = 0}, _Offset, _Label, LabelLookup) -> +branch_to_label_code(#state{}, _Offset, _Label, LabelLookup) -> error({no_available_registers, LabelLookup}). %%----------------------------------------------------------------------------- @@ -657,9 +587,10 @@ if_block( Stream2, Replacements ), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}; + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}; if_block( #state{stream_module = StreamModule, stream = Stream0} = State0, Cond, @@ -675,9 +606,10 @@ if_block( BranchOffset = OffsetAfter - BranchInstrOffset, NewBranchInstr = apply(?ASM, BranchFunc, [Reg, Operand, BranchOffset]), Stream3 = StreamModule:replace(Stream2, BranchInstrOffset, NewBranchInstr), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}. %%----------------------------------------------------------------------------- %% @doc Emit an if else block, i.e. emit a test of a condition and @@ -714,8 +646,6 @@ if_else_block( %% Build the else block StateElse = State2#state{ stream = Stream4, - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs, regs = State1#state.regs }, State3 = BlockFalseFn(StateElse), @@ -728,9 +658,10 @@ if_else_block( %% If this fails, the if/else blocks are too large 2 = byte_size(NewElseJumpInstr), Stream6 = StreamModule:replace(Stream5, ElseJumpOffset, NewElseJumpInstr), - State4 = merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs), - MergedRegs = jit_regs:merge(State2#state.regs, State3#state.regs), - State4#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State2#state.regs, State3#state.regs, ?AVAILABLE_REGS_MASK + ), + State3#state{stream = Stream6, regs = MergedRegs}. if_block_cond( #state{stream_module = StreamModule, stream = Stream0} = State0, {RegOrTuple, '<', 0} @@ -757,7 +688,7 @@ if_block_cond( end, % RISC-V: bge Reg, Val, offset (branch if Reg >= Val, i.e., NOT less than) % Load immediate into a temp register for comparison - Temp = first_avail(State0#state.available_regs), + Temp = first_avail(jit_regs:available_regs(State0#state.regs)), OffsetBefore = StreamModule:offset(Stream0), State1 = mov_immediate(State0, Temp, Val), Stream1 = State1#state.stream, @@ -769,9 +700,10 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, {bge, Reg, Temp}, BranchDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '<', Val} ) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -790,9 +722,10 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, {bge, Reg, Temp}, BranchDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {Val, '<', RegOrTuple} ) when is_integer(Val), Val >= 0, Val =< 255 -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -811,9 +744,10 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, {bge, Temp, Reg}, BranchDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {Val, '<', RegOrTuple} ) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -896,9 +830,10 @@ if_block_cond(State, {'(int)', RegOrTuple, '==', 0}) -> if_block_cond(State, {'(int)', RegOrTuple, '==', Val}) when is_integer(Val) -> if_block_cond(State, {RegOrTuple, '==', Val}); if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '!=', Val} ) when is_integer(Val) andalso Val >= 0 andalso Val =< 255 -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), Reg = case RegOrTuple of @@ -934,9 +869,10 @@ if_block_cond( if_block_cond(State, {'(int)', RegOrTuple, '!=', Val}) when is_integer(Val) -> if_block_cond(State, {RegOrTuple, '!=', Val}); if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '==', Val} ) when is_integer(Val) andalso Val >= 0 andalso Val =< 255 -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -966,9 +902,10 @@ if_block_cond( State3 = if_block_free_reg({free, RegB}, State2), {State3, {bne, RegA, RegB}, 0}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '==', Val} ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -987,9 +924,10 @@ if_block_cond( State3 = State2#state{stream = Stream2, regs = Regs2}, {State3, {bne, Reg, Temp}, BranchDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '!=', Val} ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -1011,10 +949,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, {'(bool)', RegOrTuple, '==', false} ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -1034,10 +973,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, {'(bool)', RegOrTuple, '!=', false} ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -1057,10 +997,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, {RegOrTuple, '&', Val, '!=', 0} ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -1093,10 +1034,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, {Reg, '&', 16#F, '!=', 16#F} ) when ?IS_GPR(Reg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), %% RISC-V: Special case Reg & ?TERM_IMMED_TAG_MASK != ?TERM_INTEGER_TAG I1 = ?ASM:not_(Temp, Reg), @@ -1128,19 +1070,25 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, {Reg, '&', Mask, '!=', Val} ) when ?IS_GPR(Reg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), + TempBit = reg_bit(Temp), %% RISC-V: AND with mask, then compare with value OffsetBefore = StreamModule:offset(Stream0), I1 = ?ASM:mv(Temp, Reg), Stream1 = StreamModule:append(Stream0, I1), - State1 = State0#state{stream = Stream1}, - {State2, Temp} = and_(State1#state{available_regs = AT}, {free, Temp}, Mask), + State1 = State0#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs0, TempBit) + }, + {State2, Temp} = and_(State1, {free, Temp}, Mask), Stream2 = State2#state.stream, + Regs2 = State2#state.regs, + Avail2 = jit_regs:available_regs(Regs2), %% Compare Temp with Val and branch if equal (NOT != Val) case Val of 0 -> @@ -1149,7 +1097,8 @@ if_block_cond( BranchInstr = <<16#FFFFFFFF:32/little>>, Stream3 = StreamModule:append(Stream2, BranchInstr), State3 = State2#state{ - stream = Stream3, available_regs = State2#state.available_regs bor reg_bit(Temp) + stream = Stream3, + regs = jit_regs:free_reg(Regs2, TempBit) }, {State3, {beq, Temp, zero}, BranchDelta}; _ when ?IS_GPR(Val) -> @@ -1158,24 +1107,32 @@ if_block_cond( BranchInstr = <<16#FFFFFFFF:32/little>>, Stream3 = StreamModule:append(Stream2, BranchInstr), State3 = State2#state{ - stream = Stream3, available_regs = State2#state.available_regs bor reg_bit(Temp) + stream = Stream3, + regs = jit_regs:free_reg(Regs2, TempBit) }, {State3, {beq, Temp, Val}, BranchDelta}; _ -> %% Val is an immediate - need second temp register %% Reuse the mask register for the comparison value - MaskReg = first_avail(AT), - AT2 = AT band (bnot reg_bit(MaskReg)), - State3 = mov_immediate(State2#state{available_regs = AT2}, MaskReg, Val), + MaskReg = first_avail(Avail2), + MaskRegBit = reg_bit(MaskReg), + State3 = mov_immediate( + State2#state{ + regs = jit_regs:alloc_reg(Regs2, MaskRegBit) + }, + MaskReg, + Val + ), Stream3 = State3#state.stream, + Regs3 = State3#state.regs, BranchDelta = StreamModule:offset(Stream3) - OffsetBefore, BranchInstr = <<16#FFFFFFFF:32/little>>, Stream4 = StreamModule:append(Stream3, BranchInstr), - Regs4 = jit_regs:invalidate_reg(State3#state.regs, MaskReg), + Regs4 = jit_regs:invalidate_reg(Regs3, MaskReg), + ReleaseBits = TempBit bor MaskRegBit, State4 = State3#state{ stream = Stream4, - available_regs = State3#state.available_regs bor reg_bit(Temp) bor reg_bit(MaskReg), - regs = Regs4 + regs = jit_regs:free_reg(Regs4, ReleaseBits) }, {State4, {beq, Temp, MaskReg}, BranchDelta} end; @@ -1183,10 +1140,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailRegs + regs = Regs0 } = State0, {{free, Reg} = RegTuple, '&', Mask, '!=', Val} ) when ?IS_GPR(Reg) -> + AvailRegs = jit_regs:available_regs(Regs0), %% RISC-V: AND with mask, then compare with value OffsetBefore = StreamModule:offset(Stream0), {State1, Reg} = and_(State0, RegTuple, Mask), @@ -1212,36 +1170,40 @@ if_block_cond( _ -> %% Val is an immediate - need temp register %% Reuse the mask register for the comparison value - MaskReg = first_avail(State1#state.available_regs), - AT = State1#state.available_regs band (bnot reg_bit(MaskReg)), - State2 = mov_immediate(State1#state{available_regs = AT}, MaskReg, Val), + Regs1 = State1#state.regs, + Avail1 = jit_regs:available_regs(Regs1), + MaskReg = first_avail(Avail1), + MaskRegBit = reg_bit(MaskReg), + State2 = mov_immediate( + State1#state{ + regs = jit_regs:alloc_reg(Regs1, MaskRegBit) + }, + MaskReg, + Val + ), Stream2 = State2#state.stream, + Regs2 = State2#state.regs, + Used2 = jit_regs:used_regs(Regs2), BranchDelta = StreamModule:offset(Stream2) - OffsetBefore, BranchInstr = <<16#FFFFFFFF:32/little>>, Stream3 = StreamModule:append(Stream2, BranchInstr), - Regs3 = jit_regs:invalidate_reg(State2#state.regs, MaskReg), - State3 = State2#state{stream = Stream3, available_regs = AvailRegs, regs = Regs3}, + Regs3 = jit_regs:invalidate_reg(Regs2, MaskReg), + State3 = State2#state{ + stream = Stream3, + regs = jit_regs:set_masks( + Regs3, AvailRegs, Used2 band (bnot MaskRegBit) + ) + }, State4 = if_block_free_reg(RegTuple, State3), {State4, {beq, Reg, MaskReg}, BranchDelta} end. -if_block_free_reg({free, Reg}, State0) -> - #state{available_regs = AvR0, used_regs = UR0} = State0, +if_block_free_reg({free, Reg}, #state{regs = Regs0} = State0) -> Bit = reg_bit(Reg), - AvR1 = AvR0 bor Bit, - UR1 = UR0 band (bnot Bit), - State0#state{ - available_regs = AvR1, - used_regs = UR1 - }; + State0#state{regs = jit_regs:free_reg(Regs0, Bit)}; if_block_free_reg(Reg, State0) when ?IS_GPR(Reg) -> State0. -merge_used_regs(#state{used_regs = UR} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - MergedAvail = ?AVAILABLE_REGS_MASK band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. - %%----------------------------------------------------------------------------- %% @doc Emit a shift register right by a fixed number of bits, effectively %% dividing it by 2^Shift @@ -1263,8 +1225,6 @@ shift_right( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1272,6 +1232,7 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I = ?ASM:srli(ResultReg, Reg, Shift), @@ -1280,9 +1241,7 @@ shift_right( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -1318,8 +1277,6 @@ shift_right_arith( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1327,6 +1284,7 @@ shift_right_arith( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I = ?ASM:srai(ResultReg, Reg, Shift), @@ -1335,9 +1293,7 @@ shift_right_arith( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -1355,12 +1311,13 @@ call_func_ptr( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0Mask, - used_regs = UsedRegs0Mask + regs = Regs0 } = State0, FuncPtrTuple, Args ) -> + AvailableRegs0Mask = jit_regs:available_regs(Regs0), + UsedRegs0Mask = jit_regs:used_regs(Regs0), FreeRegs = lists:flatmap( fun ({free, {ptr, Reg}}) -> [Reg]; @@ -1407,9 +1364,10 @@ call_func_ptr( % and the currently available registers SetArgsMask = (UsedRegs1Mask band (bnot RegArgsRegsMask)) bor AvailableRegs0Mask, State1 = State0#state{ - available_regs = SetArgsMask, - used_regs = ?AVAILABLE_REGS_MASK band (bnot SetArgsMask), - stream = Stream1 + stream = Stream1, + regs = jit_regs:set_masks( + Regs0, SetArgsMask, ?AVAILABLE_REGS_MASK band (bnot SetArgsMask) + ) }, ParameterRegs = parameter_regs(RegArgs0), @@ -1469,9 +1427,12 @@ call_func_ptr( end, State3 = State1#state{ - available_regs = SetArgsAvailMask, - used_regs = ?AVAILABLE_REGS_MASK band (bnot SetArgsAvailMask), - stream = Stream3 + stream = Stream3, + regs = jit_regs:set_masks( + State1#state.regs, + SetArgsAvailMask, + ?AVAILABLE_REGS_MASK band (bnot SetArgsAvailMask) + ) }, StackOffset = AlignedStackBytes, @@ -1533,9 +1494,7 @@ call_func_ptr( { State4#state{ stream = Stream8, - available_regs = AvailableRegs3Mask, - used_regs = UsedRegs2Mask, - regs = Regs1 + regs = jit_regs:set_masks(Regs1, AvailableRegs3Mask, UsedRegs2Mask) }, ResultReg }. @@ -1587,11 +1546,12 @@ set_registers_args(State0, Args, StackOffset) -> set_registers_args(State0, Args, ParamRegs, StackOffset). set_registers_args( - #state{used_regs = UsedRegsMask} = State0, + #state{regs = Regs0} = State0, Args, ParamRegs, StackOffset ) -> + UsedRegsMask = jit_regs:used_regs(Regs0), ArgsRegs = args_regs(Args), ParamMask = regs_to_mask(ParamRegs), ArgsMask = regs_to_mask(ArgsRegs), @@ -1613,8 +1573,11 @@ set_registers_args( ), State1#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), - used_regs = ParamMask bor NewUsedMask + regs = jit_regs:set_masks( + State1#state.regs, + ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), + ParamMask bor NewUsedMask + ) }. parameter_regs(Args) -> @@ -1732,11 +1695,12 @@ set_registers_args1( Stream1 = StreamModule:append(Stream0, I), State#state{stream = Stream1}; set_registers_args1( - #state{stream_module = StreamModule, stream = Stream0, available_regs = AvailRegs} = State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, X}, Reg, _StackOffset ) -> + AvailRegs = jit_regs:available_regs(Regs0), Code = ldr_y_reg(Reg, X, AvailRegs), Stream1 = StreamModule:append(Stream0, Code), State#state{stream = Stream1}; @@ -1792,10 +1756,11 @@ move_to_vm_register_emit(State0, Src, {ptr, Reg}) when is_atom(Src) -> Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), State0#state{stream = Stream1}; move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, Src, {y_reg, Y} + #state{regs = Regs0} = State0, Src, {y_reg, Y} ) when is_atom(Src) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp1)), Code = str_y_reg(Src, Y, Temp1, AT), @@ -1810,10 +1775,11 @@ move_to_vm_register_emit( State0#state{stream = Stream1, regs = Regs2}; % Source is an integer to y_reg (optimized: ldr first, then movs) move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, N, {y_reg, Y} + #state{regs = Regs0} = State0, N, {y_reg, Y} ) when is_integer(N), N >= 0, N =< 255 -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail2 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail2), @@ -1830,77 +1796,84 @@ move_to_vm_register_emit( end, State0#state{stream = Stream1, regs = Regs2}; % Source is an integer (0-255 for movs, negative values need different handling) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N), N >= 0, N =< 255 -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = ?ASM:li(Temp, N), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {imm, N}), - State1#state{available_regs = AR0, regs = Regs1}; + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = ?ASM:li(Temp, N), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); %% Handle large values using simple literal pool (branch-over pattern) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, N), - State2 = move_to_vm_register(State1, Temp, Dest), - Regs1 = jit_regs:set_contents(State2#state.regs, Temp, {imm, N}), - State2#state{available_regs = AR0, regs = Regs1}; + with_temp( + State0, + fun(StateT, Temp, _AT) -> + State1 = mov_immediate(StateT, Temp, N), + move_to_vm_register(State1, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); % Source is a VM register -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, extra}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - {BaseReg, Off} = ?X_REG(?MAX_REG), - I1 = ?LOAD_WORD(Temp, BaseReg, Off), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, ?MAX_REG}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, X}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - {XReg, X_REGOffset} = ?X_REG(X), - I1 = ?LOAD_WORD(Temp, XReg, X_REGOffset), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, X}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {ptr, Reg}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - I1 = ?LOAD_WORD(Temp, Reg, 0), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {y_reg, Y}, Dest) -> - Temp = first_avail(AR0), - AT = AR0 band (bnot reg_bit(Temp)), - Code = ldr_y_reg(Temp, Y, AT), - Stream1 = (State0#state.stream_module):append(State0#state.stream, Code), - % ldr_y_reg clobbers first_avail(AT) as a hidden temp for loading Y_REGS pointer - Regs0a = - case AT of - 0 -> State0#state.regs; - _ -> jit_regs:invalidate_reg(State0#state.regs, first_avail(AT)) +move_to_vm_register_emit(State0, {x_reg, extra}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + {BaseReg, Off} = ?X_REG(?MAX_REG), + I1 = ?LOAD_WORD(Temp, BaseReg, Off), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) end, - State0a = State0#state{ - stream = Stream1, - available_regs = AT, - regs = Regs0a - }, - State1 = move_to_vm_register(State0a, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {y_reg, Y}), - State1#state{available_regs = AR0, regs = Regs1}; + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, ?MAX_REG}) end + ); +move_to_vm_register_emit(State0, {x_reg, X}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + {XReg, X_REGOffset} = ?X_REG(X), + I1 = ?LOAD_WORD(Temp, XReg, X_REGOffset), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, X}) end + ); +move_to_vm_register_emit(State0, {ptr, Reg}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = ?LOAD_WORD(Temp, Reg, 0), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:invalidate_reg(Regs, Temp) end + ); +move_to_vm_register_emit(State0, {y_reg, Y}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, AT) -> + Code = ldr_y_reg(Temp, Y, AT), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, Code), + % ldr_y_reg clobbers first_avail(AT) as a hidden temp for the Y_REGS pointer + Regs0a = + case AT of + 0 -> StateT#state.regs; + _ -> jit_regs:invalidate_reg(StateT#state.regs, first_avail(AT)) + end, + move_to_vm_register(StateT#state{stream = Stream1, regs = Regs0a}, Temp, Dest) + end, + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {y_reg, Y}) end + ); % term_to_float move_to_vm_register_emit( #state{ stream_module = StreamModule, - available_regs = Avail, + regs = Regs0, stream = Stream0, variant = Variant } = @@ -1908,6 +1881,7 @@ move_to_vm_register_emit( {free, {ptr, Reg, 1}}, {fp_reg, F} ) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), {BaseReg, Off} = ?FP_REGS, @@ -1930,6 +1904,19 @@ move_to_vm_register_emit( Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp1), Temp2), State1#state{stream = Stream1, regs = Regs1}. +-spec with_temp( + state(), + fun((state(), native_register(), non_neg_integer()) -> state()), + fun((jit_regs:regs(), native_register()) -> jit_regs:regs()) +) -> state(). +with_temp(#state{regs = Regs0} = State0, EmitFun, ContentsFun) -> + AR0 = jit_regs:available_regs(Regs0), + Temp = first_avail(AR0), + AT = AR0 band (bnot reg_bit(Temp)), + State1 = EmitFun(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, AT), + Regs1 = jit_regs:set_available_regs(ContentsFun(State1#state.regs, Temp), AR0), + State1#state{regs = Regs1}. + %%----------------------------------------------------------------------------- %% @doc Emit a move of an array element (reg[x]) to a vm or a native register. %% @end @@ -1940,12 +1927,13 @@ move_to_vm_register_emit( %% @return Updated backend state %%----------------------------------------------------------------------------- move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?LOAD_WORD(Temp, Reg, Index * ?WORD_SIZE_BYTES), {BaseReg, Off} = ?X_REG(X), @@ -1955,12 +1943,13 @@ move_array_element( Regs2 = jit_regs:set_contents(Regs1, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?LOAD_WORD(Temp, Reg, Index * ?WORD_SIZE_BYTES), I2 = ?STORE_WORD(Dest, Temp, 0), @@ -1968,12 +1957,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail2 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail2), @@ -1987,12 +1977,13 @@ move_array_element( Regs3 = jit_regs:invalidate_reg(Regs2, Temp2), State#state{stream = Stream1, regs = Regs3}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Index, {y_reg, Y} ) when is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), I1 = ?LOAD_WORD(Reg, Reg, Index * ?WORD_SIZE_BYTES), @@ -2017,8 +2008,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2031,23 +2020,17 @@ move_array_element( {BaseReg, Off} = ?X_REG(X), I4 = ?STORE_WORD(BaseReg, IndexReg, Off), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_vm_loc(Regs0, {x_reg, X}), Regs2 = jit_regs:invalidate_reg(Regs1, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs2 + regs = jit_regs:free_reg(Regs2, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2059,28 +2042,23 @@ move_array_element( I3 = ?LOAD_WORD(IndexReg, IndexReg, 0), I4 = ?STORE_WORD(PtrReg, IndexReg, 0), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_reg(Regs0, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs1 + regs = jit_regs:free_reg(Regs1, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, {free, IndexReg}, {y_reg, Y} ) when is_atom(IndexReg) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), Temp = first_avail(AvailableRegs0), AT = AvailableRegs0 band (bnot reg_bit(Temp)), I1 = ?ASM:slli(IndexReg, IndexReg, (?WORD_SIZE_BYTES bsr 2) + 1), @@ -2089,8 +2067,6 @@ move_array_element( Code = str_y_reg(IndexReg, Y, Temp, AT), I4 = Code, Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append( Stream0, <> ), @@ -2098,10 +2074,8 @@ move_array_element( Regs2 = jit_regs:invalidate_reg(Regs1, Temp), Regs3 = jit_regs:invalidate_reg(Regs2, IndexReg), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs3 + regs = jit_regs:free_reg(Regs3, Bit) }. %% @doc move reg[x] to a vm or native register @@ -2122,13 +2096,12 @@ get_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, Index ) -> + Avail = jit_regs:available_regs(Regs0), ElemReg = first_avail(Avail), ElemBit = reg_bit(ElemReg), I1 = ?LOAD_WORD(ElemReg, Reg, Index * ?WORD_SIZE_BYTES), @@ -2137,9 +2110,7 @@ get_array_element( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ElemBit), - used_regs = UsedRegs0 bor ElemBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ElemBit) }, ElemReg }. @@ -2155,12 +2126,13 @@ move_to_array_element( Stream1 = StreamModule:append(Stream0, I1), State0#state{stream = Stream1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, ValueReg, Reg, IndexReg ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(Reg) andalso ?IS_GPR(IndexReg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:mv(Temp, IndexReg), I2 = ?ASM:slli(Temp, Temp, (?WORD_SIZE_BYTES bsr 2) + 1), @@ -2188,13 +2160,14 @@ move_to_array_element( ) when is_integer(IndexReg) andalso is_integer(Offset) andalso ?ARRAY_OFFSET_FOLD_GUARD(Offset) -> move_to_array_element(State, Value, BaseReg, ?ARRAY_OFFSET_FOLD(IndexReg, Offset)); move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, ValueReg, BaseReg, IndexReg, Offset ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:addi(Temp, IndexReg, Offset), I2 = ?ASM:slli(Temp, Temp, (?WORD_SIZE_BYTES bsr 2) + 1), @@ -2211,7 +2184,7 @@ move_to_array_element( Offset ) -> {State1, ValueReg} = copy_to_native_register(State0, Value), - Temp = first_avail(State1#state.available_regs), + Temp = first_avail(jit_regs:available_regs(State1#state.regs)), I1 = ?ASM:addi(Temp, IndexReg, Offset), I2 = ?ASM:slli(Temp, Temp, (?WORD_SIZE_BYTES bsr 2) + 1), I3 = ?ASM:add(Temp, BaseReg, Temp), @@ -2230,16 +2203,17 @@ move_to_native_register(#state{regs = Regs} = State, Value) -> case Contents =/= unknown andalso jit_regs:find_reg_with_contents(Regs, Contents) of {ok, CachedReg} -> Bit = reg_bit(CachedReg), - case State#state.used_regs band Bit of + Used = jit_regs:used_regs(Regs), + Avail = jit_regs:available_regs(Regs), + case Used band Bit of 0 -> - case State#state.available_regs band Bit of + case Avail band Bit of 0 -> move_to_native_register_emit(State, Value, Contents); _ -> { State#state{ - used_regs = State#state.used_regs bor Bit, - available_regs = State#state.available_regs band (bnot Bit) + regs = jit_regs:alloc_reg(Regs, Bit) }, CachedReg } @@ -2255,13 +2229,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, cp, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), {BaseReg, Off} = ?CP, @@ -2271,9 +2244,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2288,8 +2259,6 @@ move_to_native_register_emit( {State#state{stream = Stream1, regs = Regs1}, Reg}; move_to_native_register_emit( #state{ - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State0, Imm, @@ -2297,26 +2266,24 @@ move_to_native_register_emit( ) when is_integer(Imm) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), Regs1 = jit_regs:set_contents(Regs0, Reg, Contents), State1 = State0#state{ - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, {move_to_native_register(State1, Imm, Reg), Reg}; move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, extra}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), {BaseReg, Off} = ?X_REG(?MAX_REG), @@ -2326,9 +2293,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2336,8 +2301,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, X}, @@ -2345,6 +2308,7 @@ move_to_native_register_emit( ) when X < ?MAX_REG -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), {BaseReg, Offset} = ?X_REG(X), @@ -2354,9 +2318,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2364,13 +2326,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {y_reg, Y}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), AvailT = Avail band (bnot RegBit), @@ -2386,9 +2347,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - available_regs = AvailT, - used_regs = Used bor RegBit, - regs = Regs2 + regs = jit_regs:alloc_reg(Regs2, RegBit) }, Reg }; @@ -2396,18 +2355,17 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used + regs = Regs0 } = State, {fp_reg, F}, _Contents ) -> + Avail = jit_regs:available_regs(Regs0), RegA = first_avail(Avail), RegABit = reg_bit(RegA), Avail2 = Avail band (bnot RegABit), RegB = first_avail(Avail2), RegBBit = reg_bit(RegB), - AvailT = Avail2 band (bnot RegBBit), {BaseReg, Off} = ?FP_REGS, I1 = ?LOAD_WORD(RegB, BaseReg, Off), I2 = ?ASM:lw(RegA, RegB, F * 8), @@ -2416,7 +2374,8 @@ move_to_native_register_emit( Stream1 = StreamModule:append(Stream0, Code), { State#state{ - stream = Stream1, available_regs = AvailT, used_regs = Used bor RegABit bor RegBBit + stream = Stream1, + regs = jit_regs:alloc_reg(Regs0, RegABit bor RegBBit) }, {fp, RegA, RegB} }. @@ -2460,11 +2419,12 @@ move_to_native_register( Regs1 = jit_regs:set_contents(Regs0, RegDst, {x_reg, X}), State#state{stream = Stream1, regs = Regs1}; move_to_native_register( - #state{stream_module = StreamModule, stream = Stream0, available_regs = AT, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y}, RegDst ) -> + AT = jit_regs:available_regs(Regs0), Code = ldr_y_reg(RegDst, Y, AT), Stream1 = StreamModule:append(Stream0, Code), % ldr_y_reg clobbers first_avail(AT) as a hidden temp for loading Y_REGS pointer @@ -2497,12 +2457,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, Reg ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = ?ASM:mv(SaveReg, Reg), @@ -2512,9 +2471,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, SaveBit) }, SaveReg }; @@ -2522,12 +2479,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {ptr, Reg} ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = ?LOAD_WORD(SaveReg, Reg, 0), @@ -2536,20 +2492,48 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, SaveBit) }, SaveReg }; copy_to_native_register(State, Reg) -> move_to_native_register(State, Reg). +-if(?WORD_SIZE_BYTES =:= 4). move_to_cp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y} ) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + AvailT = Avail band (bnot reg_bit(Reg)), + % The saved cp spans two slots: y[Y] = offset word (-> ?CP), y[Y+1] = Module* + % (-> ?CP_MODULE). Copy both into ctx->cp. + I1 = ldr_y_reg(Reg, Y, AvailT), + {CpBase, CpOff} = ?CP, + I2 = ?STORE_WORD(CpBase, Reg, CpOff), + I3 = ldr_y_reg(Reg, Y + 1, AvailT), + {CpModBase, CpModOff} = ?CP_MODULE, + I4 = ?STORE_WORD(CpModBase, Reg, CpModOff), + Code = <>, + Stream1 = StreamModule:append(Stream0, Code), + % Reg ends holding y[Y+1] (the Module*); ldr_y_reg also clobbers + % first_avail(AvailT) as a hidden temp for loading the Y_REGS pointer. + Regs1a = jit_regs:invalidate_reg(Regs0, Reg), + Regs1 = + case AvailT of + 0 -> Regs1a; + _ -> jit_regs:invalidate_reg(Regs1a, first_avail(AvailT)) + end, + State#state{stream = Stream1, regs = Regs1}. +-else. +move_to_cp( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = + State, + {y_reg, Y} +) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), I1 = ldr_y_reg(Reg, Y, AvailT), @@ -2565,12 +2549,14 @@ move_to_cp( _ -> jit_regs:invalidate_reg(Regs1a, first_avail(AvailT)) end, State#state{stream = Stream1, regs = Regs1}. +-endif. increment_sp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Offset ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), {BaseReg1, Off1} = ?Y_REGS, I1 = ?LOAD_WORD(Reg, BaseReg1, Off1), @@ -2586,13 +2572,13 @@ set_continuation_to_label( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, labels = Labels, regs = Regs0 } = State, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), Offset = StreamModule:offset(Stream0), @@ -2629,11 +2615,11 @@ set_continuation_to_offset( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), OffsetRef = make_ref(), Offset = StreamModule:offset(Stream0), @@ -2656,11 +2642,10 @@ get_module_index( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), % Load module pointer from jit_state (which is in a1) @@ -2672,9 +2657,7 @@ get_module_index( { State#state{ stream = Stream1, - available_regs = Avail band (bnot RegBit), - used_regs = UsedRegs0 bor RegBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }. @@ -2715,39 +2698,59 @@ and_( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {State0#state{stream = Stream1, regs = Regs1}, Reg}; and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val ) when Val < 0 andalso Val >= -256 -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, bnot (Val)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, bnot (Val) + ), Stream1 = State1#state.stream, % RISC-V doesn't have bics, use not + and I1 = ?ASM:not_(Temp, Temp), I2 = ?ASM:and_(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, <>), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}, Reg}; + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }, + Reg + }; and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val -) when Avail =/= 0 -> +) -> + Avail = jit_regs:available_regs(Regs0), + true = Avail =/= 0, Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = ?ASM:and_(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}, Reg}; + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }, + Reg + }; and_( - #state{stream_module = StreamModule, available_regs = Avail, used_regs = UR, regs = Regs0} = + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, ?TERM_PRIMARY_CLEAR_MASK ) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I = ?ASM:andi(ResultReg, Reg, -4), @@ -2756,9 +2759,7 @@ and_( { State0#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -2778,18 +2779,19 @@ or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; or_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val), Stream1 = State1#state.stream, I = ?ASM:or_(Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. xor_( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Reg, SrcReg @@ -2808,18 +2810,19 @@ xor_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = Stat Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; xor_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val), Stream1 = State1#state.stream, I = ?ASM:xor_(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. add(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Reg, Val) when Val >= 0 andalso Val =< 255 @@ -2835,15 +2838,16 @@ add(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; -add(#state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, Reg, Val) -> +add(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val), Stream1 = State1#state.stream, I = ?ASM:add(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. mov_immediate(#state{stream_module = StreamModule, stream = Stream0} = State, Reg, Val) when Val >= -16#800, Val =< 16#7FF @@ -2873,21 +2877,23 @@ sub(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State#state{stream = Stream1, regs = Regs1}; -sub(#state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, Reg, Val) -> +sub(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val), Stream1 = State1#state.stream, I = ?ASM:sub(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Reg), Temp), + State1#state{stream = Stream2, regs = jit_regs:set_available_regs(Regs1, Avail)}. mul(State, _Reg, 1) -> State; mul(State, Reg, 2) -> shift_left(State, Reg, 1); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> +mul(#state{regs = Regs0} = State, Reg, 3) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:slli(Temp, Reg, 1), I2 = ?ASM:add(Reg, Temp, Reg), @@ -2896,7 +2902,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 4) -> shift_left(State, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> +mul(#state{regs = Regs0} = State, Reg, 5) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:slli(Temp, Reg, 2), I2 = ?ASM:add(Reg, Temp, Reg), @@ -2906,7 +2913,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> mul(State0, Reg, 6) -> State1 = mul(State0, Reg, 3), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> +mul(#state{regs = Regs0} = State, Reg, 7) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:slli(Temp, Reg, 3), I2 = ?ASM:sub(Reg, Temp, Reg), @@ -2915,7 +2923,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 8) -> shift_left(State, Reg, 3); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> +mul(#state{regs = Regs0} = State, Reg, 9) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:slli(Temp, Reg, 3), I2 = ?ASM:add(Reg, Temp, Reg), @@ -2925,7 +2934,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> mul(State0, Reg, 10) -> State1 = mul(State0, Reg, 5), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 15) -> +mul(#state{regs = Regs0} = State, Reg, 15) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = ?ASM:slli(Temp, Reg, 4), I2 = ?ASM:sub(Reg, Temp, Reg), @@ -2939,21 +2949,22 @@ mul(State, Reg, 32) -> mul(State, Reg, 64) -> shift_left(State, Reg, 6); mul( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val), Stream1 = State1#state.stream, I = ?ASM:mul(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Temp), Reg), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp), Reg), + AvailAfter = jit_regs:available_regs(State1#state.regs) bor reg_bit(Temp), State1#state{ stream = Stream2, - available_regs = State1#state.available_regs bor reg_bit(Temp), - regs = Regs1 + regs = jit_regs:set_available_regs(Regs1, AvailAfter) }; mul( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DestReg, SrcReg @@ -2975,9 +2986,10 @@ mul( %% When reductions == 0, we schedule the next process, and resume at the continuation point. %% decrement_reductions_and_maybe_schedule_next( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0 ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), % Load reduction count @@ -3022,9 +3034,8 @@ decrement_reductions_and_maybe_schedule_next( Stream3, BNEOffset, <> ), StreamN = Stream4, - State3 = merge_used_regs(State2#state{stream = StreamN}, State1#state.used_regs), %% schedule_next clobbers caller-saved regs; invalidate cache at continuation. - State3#state{regs = jit_regs:invalidate_all(State1#state.regs)}. + State2#state{stream = StreamN, regs = jit_regs:invalidate_all(State1#state.regs)}. call_or_schedule_next(State0, Label) -> {State1, RewriteOffset, TempReg} = set_cp(State0), @@ -3035,10 +3046,11 @@ call_only_or_schedule_next( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = RegsCO } = State0, Label ) -> + Avail = jit_regs:available_regs(RegsCO), Temp = first_avail(Avail), % Load reduction count (jit_state is in a1) I1 = ?ASM:lw(Temp, ?JITSTATE_REG, ?JITSTATE_REDUCTIONCOUNT_OFFSET), @@ -3111,13 +3123,43 @@ call_primitive_with_cp(State0, Primitive, Args) -> State2 = call_primitive_last(State1, Primitive, Args), rewrite_cp_offset(State2, RewriteOffset, TempReg). -set_cp(#state{available_regs = Avail, used_regs = UsedRegs} = State0) -> +-if(?WORD_SIZE_BYTES =:= 4). +set_cp(State0) -> + % 32-bit: cp spans two words. Store the Module pointer (jit_state->module) + % to ?CP_MODULE, and a placeholder offset << 2 to ?CP (the placeholder is + % patched by rewrite_cp_offset once the resume offset is known). + { + #state{stream_module = StreamModule, stream = Stream0} = State1, + ModReg + } = get_module(State0), + {CpModBase, CpModOff} = ?CP_MODULE, + IModStore = ?STORE_WORD(CpModBase, ModReg, CpModOff), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + Avail = jit_regs:available_regs(State2#state.regs), + TempReg = first_avail(Avail), + Offset = StreamModule:offset(Stream1), + % Reserve 8 bytes (two instructions) for the offset load; li may expand to + % one or two instructions, so always reserve the larger form. The 0xFFFFFFFF + % placeholders are flash-friendly (bits can only flip 1->0). + I1 = <<16#FFFFFFFF:32/little>>, + I2 = <<16#FFFFFFFF:32/little>>, + MOVOffset = Offset, + {CpBase, CpOff} = ?CP, + I3 = ?STORE_WORD(CpBase, TempReg, CpOff), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, + {State3, MOVOffset, TempReg}. +-else. +set_cp(#state{regs = RegsSC} = State0) -> + Avail = jit_regs:available_regs(RegsSC), TempReg = first_avail(Avail), TempBit = reg_bit(TempReg), % Reserve a temporary register for the offset BEFORE calling get_module_index % to avoid running out of available registers State0b = State0#state{ - available_regs = Avail band (bnot TempBit), used_regs = UsedRegs bor TempBit + regs = jit_regs:alloc_reg(RegsSC, TempBit) }, % get module index (dynamically) { @@ -3150,6 +3192,7 @@ set_cp(#state{available_regs = Avail, used_regs = UsedRegs} = State0) -> State3 = free_native_register(State2, Reg), State4 = free_native_register(State3, TempReg), {State4, MOVOffset, TempReg}. +-endif. rewrite_cp_offset( #state{stream_module = StreamModule, stream = Stream0, offset = CodeOffset} = State0, @@ -3177,10 +3220,11 @@ rewrite_cp_offset( State0#state{stream = Stream1}. set_bs( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, TermReg ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), {BaseReg1, Off1} = ?BS, I1 = ?STORE_WORD(BaseReg1, TermReg, Off1), @@ -3328,6 +3372,15 @@ ldr_y_reg(DstReg, Y, 0) when Y * ?WORD_SIZE_BYTES =< ?Y_OFFSET_LIMIT -> I2 = ?LOAD_WORD(DstReg, DstReg, Y * ?WORD_SIZE_BYTES), <>. +%% Scratch-register orderings consumed by jit_backend_regs_impl.hrl. They must be +%% defined before that file is included by jit_riscv32 / jit_riscv64; since this +%% file is itself included earlier, defining them here suffices. first_avail uses +%% only the temporaries (t0-t6); mask_to_list additionally covers the argument +%% registers (a0-a7), which appear in used masks during calls. +-define(FIRST_AVAIL_REGS, [t6, t5, t4, t3, t2, t1, t0]). +-define(MASK_TO_LIST_REGS, [t6, t5, t4, t3, t2, t1, t0, a7, a6, a5, a4, a3, a2, a1, a0]). +-define(JITSTATE_ARG_REG, jit_state). + reg_bit(a0) -> ?REG_BIT_A0; reg_bit(a1) -> ?REG_BIT_A1; reg_bit(a2) -> ?REG_BIT_A2; @@ -3352,76 +3405,6 @@ regs_to_mask([offset | T]) -> regs_to_mask(T); regs_to_mask([stack | T]) -> regs_to_mask(T); regs_to_mask([Reg | T]) -> reg_bit(Reg) bor regs_to_mask(T). -%% first_avail returns the first available register from a bitmask. -%% Order matches AVAILABLE_REGS = [t6, t5, t4, t3, t2, t1, t0] -first_avail(Mask) when Mask band ?REG_BIT_T6 =/= 0 -> t6; -first_avail(Mask) when Mask band ?REG_BIT_T5 =/= 0 -> t5; -first_avail(Mask) when Mask band ?REG_BIT_T4 =/= 0 -> t4; -first_avail(Mask) when Mask band ?REG_BIT_T3 =/= 0 -> t3; -first_avail(Mask) when Mask band ?REG_BIT_T2 =/= 0 -> t2; -first_avail(Mask) when Mask band ?REG_BIT_T1 =/= 0 -> t1; -first_avail(Mask) when Mask band ?REG_BIT_T0 =/= 0 -> t0. - -%% Convert bitmask to list, covering all register bits. -mask_to_list(0) -> []; -mask_to_list(Mask) -> mask_to_list_t6(Mask). - -mask_to_list_t6(Mask) when Mask band ?REG_BIT_T6 =/= 0 -> [t6 | mask_to_list_t5(Mask)]; -mask_to_list_t6(Mask) -> mask_to_list_t5(Mask). -mask_to_list_t5(Mask) when Mask band ?REG_BIT_T5 =/= 0 -> [t5 | mask_to_list_t4(Mask)]; -mask_to_list_t5(Mask) -> mask_to_list_t4(Mask). -mask_to_list_t4(Mask) when Mask band ?REG_BIT_T4 =/= 0 -> [t4 | mask_to_list_t3(Mask)]; -mask_to_list_t4(Mask) -> mask_to_list_t3(Mask). -mask_to_list_t3(Mask) when Mask band ?REG_BIT_T3 =/= 0 -> [t3 | mask_to_list_t2(Mask)]; -mask_to_list_t3(Mask) -> mask_to_list_t2(Mask). -mask_to_list_t2(Mask) when Mask band ?REG_BIT_T2 =/= 0 -> [t2 | mask_to_list_t1(Mask)]; -mask_to_list_t2(Mask) -> mask_to_list_t1(Mask). -mask_to_list_t1(Mask) when Mask band ?REG_BIT_T1 =/= 0 -> [t1 | mask_to_list_t0(Mask)]; -mask_to_list_t1(Mask) -> mask_to_list_t0(Mask). -mask_to_list_t0(Mask) when Mask band ?REG_BIT_T0 =/= 0 -> [t0 | mask_to_list_a7(Mask)]; -mask_to_list_t0(Mask) -> mask_to_list_a7(Mask). -mask_to_list_a7(Mask) when Mask band ?REG_BIT_A7 =/= 0 -> [a7 | mask_to_list_a6(Mask)]; -mask_to_list_a7(Mask) -> mask_to_list_a6(Mask). -mask_to_list_a6(Mask) when Mask band ?REG_BIT_A6 =/= 0 -> [a6 | mask_to_list_a5(Mask)]; -mask_to_list_a6(Mask) -> mask_to_list_a5(Mask). -mask_to_list_a5(Mask) when Mask band ?REG_BIT_A5 =/= 0 -> [a5 | mask_to_list_a4(Mask)]; -mask_to_list_a5(Mask) -> mask_to_list_a4(Mask). -mask_to_list_a4(Mask) when Mask band ?REG_BIT_A4 =/= 0 -> [a4 | mask_to_list_a3(Mask)]; -mask_to_list_a4(Mask) -> mask_to_list_a3(Mask). -mask_to_list_a3(Mask) when Mask band ?REG_BIT_A3 =/= 0 -> [a3 | mask_to_list_a2(Mask)]; -mask_to_list_a3(Mask) -> mask_to_list_a2(Mask). -mask_to_list_a2(Mask) when Mask band ?REG_BIT_A2 =/= 0 -> [a2 | mask_to_list_a1(Mask)]; -mask_to_list_a2(Mask) -> mask_to_list_a1(Mask). -mask_to_list_a1(Mask) when Mask band ?REG_BIT_A1 =/= 0 -> [a1 | mask_to_list_a0(Mask)]; -mask_to_list_a1(Mask) -> mask_to_list_a0(Mask). -mask_to_list_a0(Mask) when Mask band ?REG_BIT_A0 =/= 0 -> [a0]; -mask_to_list_a0(_Mask) -> []. - -args_regs(Args) -> - lists:map( - fun - ({free, {ptr, Reg}}) -> Reg; - ({free, Reg}) when is_atom(Reg) -> Reg; - ({free, Imm}) when is_integer(Imm) -> imm; - (offset) -> imm; - (ctx) -> ?CTX_REG; - (jit_state) -> jit_state; - (jit_state_tail_call) -> jit_state; - (stack) -> stack; - (Reg) when is_atom(Reg) -> Reg; - (Imm) when is_integer(Imm) -> imm; - ({ptr, Reg}) -> Reg; - ({x_reg, _}) -> ?CTX_REG; - ({y_reg, _}) -> ?CTX_REG; - ({fp_reg, _}) -> ?CTX_REG; - ({free, {x_reg, _}}) -> ?CTX_REG; - ({free, {y_reg, _}}) -> ?CTX_REG; - ({free, {fp_reg, _}}) -> ?CTX_REG; - ({avm_int64_t, _}) -> imm - end, - Args - ). - %%----------------------------------------------------------------------------- %% @doc Add a label at the current offset. %% @end diff --git a/libs/jit/src/jit_wasm32.erl b/libs/jit/src/jit_wasm32.erl index f8e00effaa..5cfde850f3 100644 --- a/libs/jit/src/jit_wasm32.erl +++ b/libs/jit/src/jit_wasm32.erl @@ -85,6 +85,9 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -131,10 +134,13 @@ %% Context struct offsets (32-bit architecture, same as armv6m/riscv32) -define(CTX_E_OFFSET, 16#14). -define(CTX_X_OFFSET, 16#18). +%% ctx->cp is a 64-bit cp_t spanning two words (little-endian): low word at +%% 0x70 = offset << 2 (?CTX_CP_OFFSET), high word at 0x74 = Module* (?CTX_CP_MODULE_OFFSET). -define(CTX_CP_OFFSET, 16#5C). --define(CTX_FR_OFFSET, 16#60). --define(CTX_BS_OFFSET, 16#64). --define(CTX_BS_OFFSET_OFFSET, 16#68). +-define(CTX_CP_MODULE_OFFSET, 16#60). +-define(CTX_FR_OFFSET, 16#64). +-define(CTX_BS_OFFSET, 16#68). +-define(CTX_BS_OFFSET_OFFSET, 16#6C). %% JITState struct offsets -define(JITSTATE_MODULE_OFFSET, 16#0). @@ -160,8 +166,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), non_neg_integer()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), max_scratch :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), @@ -220,12 +224,10 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, max_scratch = ?NUM_SCRATCH_LOCALS, labels = #{}, variant = Variant, - regs = jit_regs:new(), + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0), func_bodies = [], current_body = <<>>, current_label = undefined, @@ -265,11 +267,13 @@ flush(#state{stream_module = StreamModule, stream = Stream0} = State) -> debugger(State) -> emit(State, jit_wasm32_asm:unreachable()). +%% Native-register allocation bookkeeping. Flow through jit_regs so the masks +%% live inside jit_regs:regs() rather than #state{}. -spec used_regs(state()) -> [wasm_local()]. -used_regs(#state{used_regs = Used}) -> mask_to_locals(Used). +used_regs(#state{regs = Regs}) -> mask_to_locals(jit_regs:used_regs(Regs)). -spec available_regs(state()) -> [wasm_local()]. -available_regs(#state{available_regs = Available}) -> mask_to_locals(Available). +available_regs(#state{regs = Regs}) -> mask_to_locals(jit_regs:available_regs(Regs)). -spec free_native_registers(state(), [value()]) -> state(). free_native_registers(State, []) -> @@ -279,17 +283,12 @@ free_native_registers(State, [Val | Rest]) -> free_native_registers(State1, Rest). -spec free_native_register(state(), value()) -> state(). -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Local -) when is_atom(Local) -> +free_native_register(#state{regs = Regs} = State, Local) when is_atom(Local) -> LocalIdx = jit_wasm32_asm:local_index(Local), case LocalIdx >= ?FIRST_SCRATCH_LOCAL of true -> Bit = local_bit(LocalIdx), - State#state{ - available_regs = Available0 bor Bit, used_regs = Used0 band (bnot Bit) - }; + State#state{regs = jit_regs:free_reg(Regs, Bit)}; false -> State end; @@ -299,10 +298,10 @@ free_native_register(State, _Other) -> State. -spec assert_all_native_free(state()) -> ok. -assert_all_native_free(#state{max_scratch = MS} = State) -> - 0 = State#state.used_regs, +assert_all_native_free(#state{max_scratch = MS, regs = Regs}) -> + 0 = jit_regs:used_regs(Regs), AllFree = (1 bsl MS) - 1, - AllFree = State#state.available_regs, + AllFree = jit_regs:available_regs(Regs), ok. %%============================================================================= @@ -362,10 +361,9 @@ call_primitive(State0, Primitive, Args) -> call_primitive_last(State0, Primitive, Args) -> State1 = emit_call_primitive(State0, Primitive, Args, none, true), AllFree = (1 bsl State1#state.max_scratch) - 1, + Regs1 = jit_regs:unreachable(State1#state.regs), State1#state{ - available_regs = AllFree, - used_regs = 0, - regs = jit_regs:unreachable(State1#state.regs) + regs = jit_regs:set_masks(Regs1, AllFree, 0) }. call_primitive_with_cp(State0, Primitive, Args) -> @@ -390,14 +388,13 @@ call_primitive_with_cp(State0, Primitive, Args) -> NewFuncBodies = [{PrevLabel, FinalizedBody} | FuncBodies], ContLabelOff = JumpTableStart + ContLabel * ?JUMP_TABLE_ENTRY_SIZE, AllFree = (1 bsl State3#state.max_scratch) - 1, + Regs1 = jit_regs:invalidate_all(State3#state.regs), State3#state{ func_bodies = NewFuncBodies, current_body = <<>>, current_label = ContLabel, labels = Labels#{ContLabel => ContLabelOff}, - available_regs = AllFree, - used_regs = 0, - regs = jit_regs:invalidate_all(State3#state.regs) + regs = jit_regs:set_masks(Regs1, AllFree, 0) }. %%============================================================================= @@ -405,7 +402,7 @@ call_primitive_with_cp(State0, Primitive, Args) -> %%============================================================================= return_if_not_equal_to_ctx( - #state{available_regs = Available0, used_regs = Used0} = State0, + #state{regs = Regs0} = State0, {free, Local} ) -> Code = << @@ -419,12 +416,10 @@ return_if_not_equal_to_ctx( (jit_wasm32_asm:end_())/binary >>, Bit = local_bit(Local), - Regs1 = jit_regs:invalidate_reg(State0#state.regs, Local), + Regs1 = jit_regs:invalidate_reg(Regs0, Local), State1 = emit(State0, Code), State1#state{ - available_regs = Available0 bor Bit, - used_regs = Used0 band (bnot Bit), - regs = Regs1 + regs = jit_regs:free_reg(Regs1, Bit) }. jump_to_label(State0, Label) -> @@ -479,10 +474,9 @@ jump_to_continuation(State0, {free, OffsetLocal}) -> >>, State1 = emit(State0, Code), AllFree = (1 bsl State1#state.max_scratch) - 1, + Regs1 = jit_regs:unreachable(State1#state.regs), State1#state{ - available_regs = AllFree, - used_regs = 0, - regs = jit_regs:unreachable(State1#state.regs) + regs = jit_regs:set_masks(Regs1, AllFree, 0) }. %%============================================================================= @@ -497,15 +491,17 @@ if_block(State0, {'and', CondList}, BlockFn) -> State2 = emit(State1, jit_wasm32_asm:if_(jit_wasm32_asm:blocktype_void())), State3 = BlockFn(State2), State4 = emit(State3, jit_wasm32_asm:end_()), - MergedRegs = jit_regs:merge(State1#state.regs, State4#state.regs), - merge_used_regs(State4#state{regs = MergedRegs}, State1#state.used_regs); + AllRegsMask = (1 bsl State4#state.max_scratch) - 1, + MergedRegs = merge_branch_regs(State1#state.regs, State4#state.regs, AllRegsMask), + State4#state{regs = MergedRegs}; if_block(State0, Cond, BlockFn) -> State1 = emit_condition(State0, Cond), State2 = emit(State1, jit_wasm32_asm:if_(jit_wasm32_asm:blocktype_void())), State3 = BlockFn(State2), State4 = emit(State3, jit_wasm32_asm:end_()), - MergedRegs = jit_regs:merge(State1#state.regs, State4#state.regs), - merge_used_regs(State4#state{regs = MergedRegs}, State1#state.used_regs). + AllRegsMask = (1 bsl State4#state.max_scratch) - 1, + MergedRegs = merge_branch_regs(State1#state.regs, State4#state.regs, AllRegsMask), + State4#state{regs = MergedRegs}. -spec if_else_block(state(), condition(), fun((state()) -> state()), fun((state()) -> state())) -> state(). @@ -515,13 +511,23 @@ if_else_block(State0, Cond, BlockTrueFn, BlockFalseFn) -> State3 = BlockTrueFn(State2), State4 = emit(State3, jit_wasm32_asm:else_()), StateElse = State4#state{ - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs + regs = jit_regs:set_masks( + State4#state.regs, + jit_regs:available_regs(State1#state.regs), + jit_regs:used_regs(State1#state.regs) + ) }, State5 = BlockFalseFn(StateElse), State6 = emit(State5, jit_wasm32_asm:end_()), - MergedRegs = jit_regs:merge(State3#state.regs, State5#state.regs), - merge_used_regs(State6#state{regs = MergedRegs}, State3#state.used_regs). + AllRegsMask = (1 bsl State6#state.max_scratch) - 1, + MergedRegs = merge_branch_regs(State3#state.regs, State5#state.regs, AllRegsMask), + State6#state{regs = MergedRegs}. + +merge_branch_regs(Regs1, Regs2, AllRegsMask) -> + Merged = jit_regs:merge(Regs1, Regs2), + Used = (jit_regs:used_regs(Regs1) bor jit_regs:used_regs(Regs2)) band AllRegsMask, + Available = AllRegsMask band (bnot Used), + jit_regs:set_masks(Merged, Available, Used). %%============================================================================= %% Arithmetic and bitwise operations @@ -800,24 +806,32 @@ move_to_native_register(State0, Value, Local) -> State1#state{regs = Regs1}. move_to_cp(State0, {y_reg, Y}) -> - %% Load y register and store to ctx->cp - {State1, TempLocal} = alloc_local(State0), + %% The saved cp spans two slots: e[Y] = offset word (-> ctx->cp), e[Y+1] = + %% Module* (-> ctx->cp_module). Copy both into ctx->cp. + {State1, BaseLocal} = alloc_local(State0), + {State2, TempLocal} = alloc_local(State1), Code = << - %% Load ctx->e + %% BaseLocal = ctx->e (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:i32_load(2, ?CTX_E_OFFSET))/binary, + (jit_wasm32_asm:local_set(BaseLocal))/binary, + %% ctx->cp = e[Y] (offset word) + (jit_wasm32_asm:local_get(BaseLocal))/binary, + (jit_wasm32_asm:i32_load(2, Y * 4))/binary, (jit_wasm32_asm:local_set(TempLocal))/binary, - %% Load e[Y] + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:local_get(TempLocal))/binary, - (jit_wasm32_asm:i32_load(2, Y * 4))/binary, + (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary, + %% ctx->cp_module = e[Y+1] (Module*) + (jit_wasm32_asm:local_get(BaseLocal))/binary, + (jit_wasm32_asm:i32_load(2, (Y + 1) * 4))/binary, (jit_wasm32_asm:local_set(TempLocal))/binary, - %% Store to ctx->cp (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:local_get(TempLocal))/binary, - (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary + (jit_wasm32_asm:i32_store(2, ?CTX_CP_MODULE_OFFSET))/binary >>, - State2 = emit(State1, Code), - free_native_register(State2, TempLocal). + State3 = emit(State2, Code), + free_native_register(free_native_register(State3, TempLocal), BaseLocal). move_array_element(State0, Base, Index, {ptr, Dest}) when is_integer(Index) -> %% Load Base[Index] and store to [Dest] @@ -1077,14 +1091,13 @@ decrement_reductions_and_maybe_schedule_next(State0) -> NewFuncBodies = [{PrevLabel, FinalizedBody} | FuncBodies], ContLabelOff = JumpTableStart + ContLabel * ?JUMP_TABLE_ENTRY_SIZE, AllFree = (1 bsl State9#state.max_scratch) - 1, + Regs1 = jit_regs:invalidate_all(State9#state.regs), State9#state{ func_bodies = NewFuncBodies, current_body = <<>>, current_label = ContLabel, labels = Labels#{ContLabel => ContLabelOff}, - available_regs = AllFree, - used_regs = 0, - regs = jit_regs:invalidate_all(State9#state.regs) + regs = jit_regs:set_masks(Regs1, AllFree, 0) }. call_or_schedule_next(State0, Label) -> @@ -1115,14 +1128,13 @@ call_or_schedule_next(State0, Label) -> NewFuncBodies = [{PrevLabel, FinalizedBody} | FuncBodies], ContLabelOff = JumpTableStart + ContLabel * ?JUMP_TABLE_ENTRY_SIZE, AllFree = (1 bsl State3#state.max_scratch) - 1, + Regs1 = jit_regs:invalidate_all(State3#state.regs), State3#state{ func_bodies = NewFuncBodies, current_body = <<>>, current_label = ContLabel, labels = Labels#{ContLabel => ContLabelOff}, - available_regs = AllFree, - used_regs = 0, - regs = jit_regs:invalidate_all(State3#state.regs) + regs = jit_regs:set_masks(Regs1, AllFree, 0) }. call_only_or_schedule_next(State0, Label) -> @@ -1176,8 +1188,10 @@ call_func_ptr(State0, FuncPtrTuple, Args) -> State4 = emit(State3, jit_wasm32_asm:call_indirect(TypeIdx, 0)), %% Store result State5 = emit(State4, jit_wasm32_asm:local_set(ResultLocal)), - Regs1 = jit_regs:invalidate_all(State0#state.regs), State6 = free_func_ptr(State5, FuncPtrTuple), + %% Invalidate contents tracking but keep the masks from State6: they + %% account for ResultLocal being allocated and the freed arguments. + Regs1 = jit_regs:invalidate_all(State6#state.regs), {State6#state{regs = Regs1}, ResultLocal}. %%============================================================================= @@ -1322,9 +1336,7 @@ add_label( current_body = <<>>, current_label = ContLabel, labels = Labels#{ContLabel => ContLabelOff}, - available_regs = AllFree, - used_regs = 0, - regs = Regs1 + regs = jit_regs:set_masks(Regs1, AllFree, 0) }; add_label( #state{ @@ -1357,28 +1369,31 @@ emit(#state{current_body = Body} = State, Code) -> State#state{current_body = <>}. %% Allocate a scratch local, extending the pool if exhausted. -alloc_local(#state{available_regs = 0, used_regs = Used, max_scratch = MaxScratch} = State) -> - LocalIdx = ?FIRST_SCRATCH_LOCAL + MaxScratch, - Bit = 1 bsl MaxScratch, - LocalAtom = index_to_local(LocalIdx), - { - State#state{ - used_regs = Used bor Bit, - max_scratch = MaxScratch + 1 - }, - LocalAtom - }; -alloc_local(#state{available_regs = Available, used_regs = Used} = State) -> - LocalIdx = first_avail_local(Available), - Bit = local_bit(LocalIdx), - LocalAtom = index_to_local(LocalIdx), - { - State#state{ - available_regs = Available band (bnot Bit), - used_regs = Used bor Bit - }, - LocalAtom - }. +alloc_local(#state{regs = Regs, max_scratch = MaxScratch} = State) -> + case jit_regs:available_regs(Regs) of + 0 -> + %% Pool exhausted: grow it by one local. The new local's bit is + %% beyond the current available mask, so alloc_reg/2 marks it used + %% (available stays 0). + LocalIdx = ?FIRST_SCRATCH_LOCAL + MaxScratch, + Bit = 1 bsl MaxScratch, + LocalAtom = index_to_local(LocalIdx), + { + State#state{ + regs = jit_regs:alloc_reg(Regs, Bit), + max_scratch = MaxScratch + 1 + }, + LocalAtom + }; + Available -> + LocalIdx = first_avail_local(Available), + Bit = local_bit(LocalIdx), + LocalAtom = index_to_local(LocalIdx), + { + State#state{regs = jit_regs:alloc_reg(Regs, Bit)}, + LocalAtom + } + end. %% Get the first available local from bitmask first_avail_local(Mask) -> @@ -1426,12 +1441,6 @@ local_bit(LocalAtom) when is_atom(LocalAtom) -> local_bit(LocalIdx) when is_integer(LocalIdx), LocalIdx >= ?FIRST_SCRATCH_LOCAL -> 1 bsl (LocalIdx - ?FIRST_SCRATCH_LOCAL). -merge_used_regs(#state{used_regs = UR, max_scratch = MS} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - AllFree = (1 bsl MS) - 1, - MergedAvail = AllFree band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. - emit_value_to_stack(cp) -> << (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, @@ -1483,30 +1492,61 @@ emit_set_continuation_for_label(State, _RefLabel) -> emit(State, Code). %% Emit code to set up CP (continuation pointer for returns). -%% CP format: (module_index << 24) | (label_offset << 2) +%% cp is two words: ctx->cp = label_offset << 2, ctx->cp_module = Module*. emit_set_cp_for_label(State0, Label) -> LabelOffset = Label * ?JUMP_TABLE_ENTRY_SIZE, emit_set_cp_for_offset(State0, LabelOffset). emit_set_cp_for_offset(State0, LabelOffset) -> - {State1, ModIdxLocal} = get_module_index(State0), - State2 = shift_left(State1, ModIdxLocal, 24), - {State3, OffsetLocal} = alloc_local(State2), + {State1, ModLocal} = get_module(State0), Code = << + %% ctx->cp = label_offset << 2 + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, (jit_wasm32_asm:i32_const(LabelOffset bsl 2))/binary, - (jit_wasm32_asm:local_set(OffsetLocal))/binary + (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary, + %% ctx->cp_module = Module* + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, + (jit_wasm32_asm:local_get(ModLocal))/binary, + (jit_wasm32_asm:i32_store(2, ?CTX_CP_MODULE_OFFSET))/binary >>, - State4 = emit(State3, Code), - State5 = or_(State4, ModIdxLocal, OffsetLocal), - State6 = free_native_register(State5, OffsetLocal), - %% Store CP to ctx->cp - Code2 = << + State2 = emit(State1, Code), + free_native_register(State2, ModLocal). + +%% Load the current module pointer (jit_state->module) into a fresh local. +get_module(State0) -> + {State1, Local} = alloc_local(State0), + Code = << + (jit_wasm32_asm:local_get(?JITSTATE_LOCAL))/binary, + (jit_wasm32_asm:i32_load(2, ?JITSTATE_MODULE_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary + >>, + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. + +%% Load the Module pointer stored in ctx->cp_module into a fresh local. +get_cp_module(State0) -> + {State1, Local} = alloc_local(State0), + Code = << + (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, + (jit_wasm32_asm:i32_load(2, ?CTX_CP_MODULE_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary + >>, + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. + +%% Load the offset word (offset << 2) stored in ctx->cp into a fresh local. +get_cp_offset(State0) -> + {State1, Local} = alloc_local(State0), + Code = << (jit_wasm32_asm:local_get(?CTX_LOCAL))/binary, - (jit_wasm32_asm:local_get(ModIdxLocal))/binary, - (jit_wasm32_asm:i32_store(2, ?CTX_CP_OFFSET))/binary + (jit_wasm32_asm:i32_load(2, ?CTX_CP_OFFSET))/binary, + (jit_wasm32_asm:local_set(Local))/binary >>, - State7 = emit(State6, Code2), - free_native_register(State7, ModIdxLocal). + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Local), + State2 = emit(State1, Code), + {State2#state{regs = Regs1}, Local}. emit_call_primitive(State0, Primitive, Args, ResultLocal, IsTailCall) -> WasmArgCount = count_wasm_args(Args), diff --git a/libs/jit/src/jit_x86_64.erl b/libs/jit/src/jit_x86_64.erl index 1f06c7482d..7bc907b974 100644 --- a/libs/jit/src/jit_x86_64.erl +++ b/libs/jit/src/jit_x86_64.erl @@ -137,8 +137,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), non_neg_integer()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), regs :: jit_regs:regs() @@ -263,11 +261,9 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, labels = #{}, variant = Variant, - regs = jit_regs:new() + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0) }. %%----------------------------------------------------------------------------- @@ -312,70 +308,14 @@ debugger(#state{stream_module = StreamModule, stream = Stream0} = State) -> Stream1 = StreamModule:append(Stream0, <<16#CC>>), State#state{stream = Stream1}. -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently used native registers. This is used for -%% debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of used registers -%%----------------------------------------------------------------------------- --spec used_regs(state()) -> [x86_64_register()]. -used_regs(#state{used_regs = Used}) -> mask_to_list(Used). - -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently available native scratch registers. This -%% is used for debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of available registers -%%----------------------------------------------------------------------------- --spec available_regs(state()) -> [x86_64_register()]. -available_regs(#state{available_regs = Available}) -> mask_to_list(Available). - -%%----------------------------------------------------------------------------- -%% @doc Free native registers. The passed list of registers can contain -%% registers, pointer to registers or other values that are ignored. -%% @end -%% @param State current backend state -%% @param Regs list of registers or other values -%% @return The updated backend state -%%----------------------------------------------------------------------------- --spec free_native_registers(state(), [value()]) -> state(). -free_native_registers(State, []) -> - State; -free_native_registers(State, [Reg | Rest]) -> - State1 = free_native_register(State, Reg), - free_native_registers(State1, Rest). - --spec free_native_register(state(), value()) -> state(). -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Reg -) when - is_atom(Reg) --> - Bit = reg_bit(Reg), - State#state{ - available_regs = Available0 bor Bit, - used_regs = Used0 band (bnot Bit) - }; -free_native_register(State, {ptr, Reg}) -> - free_native_register(State, Reg); -free_native_register(State, _Other) -> - State. - -%%----------------------------------------------------------------------------- -%% @doc Assert that all native scratch registers are available. This is used -%% for debugging and not in production. -%% @end -%% @param State current backend state -%% @return ok -%%----------------------------------------------------------------------------- --spec assert_all_native_free(state()) -> ok. -assert_all_native_free(State) -> - 0 = State#state.used_regs, - ?AVAILABLE_REGS_MASK = State#state.available_regs, - ok. +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1, +%% first_avail/1, mask_to_list/1, args_regs/1, prepare_call_scratch/1) is shared +%% across the register-based backends and flows through jit_regs. +-define(FIRST_AVAIL_REGS, [rax, r11, r10, r9, r8, rcx]). +-define(MASK_TO_LIST_REGS, [rcx, r8, r9, r10, r11, rax]). +-define(JITSTATE_ARG_REG, ?JITSTATE_REG). +-include("jit_backend_regs_impl.hrl"). %%----------------------------------------------------------------------------- %% @doc Emit the jump table at the beginning of the module. Branches will be @@ -500,12 +440,12 @@ call_primitive( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs + regs = Regs0 } = State, Primitive, Args ) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), % We need a register for the function pointer that should not be used as a parameter ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), @@ -529,9 +469,10 @@ call_primitive( call_func_ptr( State#state{ stream = Stream1, - available_regs = AvailableRegs0 band (bnot TempBit), - used_regs = UsedRegs bor TempBit, - regs = jit_regs:invalidate_reg(State#state.regs, Temp) + regs = jit_regs:alloc_reg( + jit_regs:invalidate_reg(Regs0, Temp), + TempBit + ) }, {free, Temp}, Args @@ -566,9 +507,9 @@ call_primitive_last( Stream1 = StreamModule:append(Stream0, Call), State0#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State0#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State0#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) }; call_primitive_last( #state{ @@ -581,16 +522,15 @@ call_primitive_last( % We need a register for the function pointer that should not be used as a parameter % Since we're not returning, we can use all scratch registers except % registers used for parameters - ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), - ArgsRegs = args_regs(Args), - ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), - ArgsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), - ScratchMask = - ?AVAILABLE_REGS_MASK band (bnot (ArgsMask bor ParamMask)), - Temp = first_avail(ScratchMask), - TempBit = reg_bit(Temp), - AvailableRegs1 = ScratchMask band (bnot TempBit), - UsedRegs = ?AVAILABLE_REGS_MASK band (bnot AvailableRegs1), + #{ + temp := Temp, + available_mask := AvailableRegs1, + used_mask := UsedRegs, + param_regs := ParamRegs, + args_regs := ArgsRegs, + param_mask := ParamMask, + args_mask := ArgsMask + } = prepare_call_scratch(Args), PrepCall = case Primitive of 0 -> @@ -602,9 +542,11 @@ call_primitive_last( State1 = set_args2( State0#state{ stream = Stream1, - available_regs = AvailableRegs1, - used_regs = UsedRegs, - regs = jit_regs:invalidate_reg(State0#state.regs, Temp) + regs = jit_regs:set_masks( + jit_regs:invalidate_reg(State0#state.regs, Temp), + AvailableRegs1, + UsedRegs + ) }, Args, ParamRegs, @@ -617,9 +559,9 @@ call_primitive_last( Stream3 = StreamModule:append(Stream2, Call), State1#state{ stream = Stream3, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State1#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State1#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) }. %%----------------------------------------------------------------------------- @@ -635,8 +577,7 @@ return_if_not_equal_to_ctx( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State, {free, Reg} ) -> @@ -652,9 +593,7 @@ return_if_not_equal_to_ctx( RegBit = reg_bit(Reg), State#state{ stream = Stream1, - available_regs = AvailableRegs0 bor RegBit, - used_regs = UsedRegs0 band (bnot RegBit), - regs = State#state.regs + regs = jit_regs:free_reg(Regs0, RegBit) }. %%----------------------------------------------------------------------------- @@ -714,10 +653,11 @@ jump_to_continuation( stream_module = StreamModule, stream = Stream0, offset = BaseOffset, - available_regs = Avail + regs = Regs0 } = State, {free, OffsetReg} ) -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), % Calculate absolute address: native_code_base + target_offset % where native_code_base = current_pc + (BaseOffset - CurrentStreamOffset) @@ -738,9 +678,9 @@ jump_to_continuation( % Free all registers since this is a tail jump State#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(Regs0), ?AVAILABLE_REGS_MASK, 0 + ) }. %%----------------------------------------------------------------------------- @@ -781,11 +721,12 @@ if_block( Stream2, Replacements ), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), %% At the merge point, only keep register tracking that is consistent %% in both the taken (State2) and not-taken (State1) paths - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}; + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}; if_block( #state{stream_module = StreamModule, stream = Stream0} = State0, Cond, @@ -801,9 +742,10 @@ if_block( Stream3 = StreamModule:replace(Stream2, Offset + ReplaceDelta, << (OffsetAfter - OffsetAfterCond) >>), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}. %%----------------------------------------------------------------------------- %% @doc Emit an if else block, i.e. emit a test of a condition and @@ -838,8 +780,6 @@ if_else_block( >>), StateElse = State2#state{ stream = Stream4, - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs, regs = State1#state.regs }, State3 = BlockFalseFn(StateElse), @@ -848,10 +788,11 @@ if_else_block( Stream6 = StreamModule:replace(Stream5, ElseJumpOffset + RelocJMPOffset, << (OffsetFinal - OffsetAfter) >>), - State4 = merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs), %% Merge register tracking from both branches (true=State2, false=State3) - MergedRegs = jit_regs:merge(State2#state.regs, State3#state.regs), - State4#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State2#state.regs, State3#state.regs, ?AVAILABLE_REGS_MASK + ), + State3#state{stream = Stream6, regs = MergedRegs}. -spec if_block_cond(state(), condition()) -> {state(), non_neg_integer()}. if_block_cond(#state{stream_module = StreamModule} = State0, Cond) -> @@ -884,8 +825,9 @@ if_block_cond0(State0, {Value, '<', RegOrTuple}) when ?IS_SINT32_T(Value) -> {State1, <>, byte_size(I1) + RelocJLEOffset}; % Catch-all for large values outside SINT32_T range if_block_cond0( - #state{available_regs = Avail, regs = Regs0} = State0, {Value, '<', RegOrTuple} + #state{regs = Regs0} = State0, {Value, '<', RegOrTuple} ) when is_integer(Value) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -920,8 +862,9 @@ if_block_cond0(State0, {RegOrTuple, '<', RegB}) when is_atom(RegB) -> {State1, <>, byte_size(I1) + RelocJGEOffset}; % Catch-all for large values outside SINT32_T range if_block_cond0( - #state{available_regs = Avail, regs = Regs0} = State0, {RegOrTuple, '<', Value} + #state{regs = Regs0} = State0, {RegOrTuple, '<', Value} ) when is_integer(Value) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -988,9 +931,10 @@ if_block_cond0( State1 = if_block_free_reg(RegOrTuple, State0), {State1, <>, byte_size(I1) + RelocJZOffset}; if_block_cond0( - #state{available_regs = Avail, regs = Regs0} = State0, + #state{regs = Regs0} = State0, {RegOrTuple, '!=', Val} ) when is_integer(Val) orelse ?IS_GPR(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -1030,9 +974,10 @@ if_block_cond0( State1 = if_block_free_reg(RegOrTuple, State0), {State1, <>, byte_size(I1) + RelocJZOffset}; if_block_cond0( - #state{available_regs = Avail, regs = Regs0} = State0, + #state{regs = Regs0} = State0, {RegOrTuple, '==', Val} ) when is_integer(Val) orelse ?IS_GPR(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Reg = case RegOrTuple of @@ -1112,7 +1057,7 @@ if_block_cond0(#state{regs = Regs0} = State0, {{free, Reg} = RegTuple, '&', Mask State1 = if_block_free_reg(RegTuple, State0#state{regs = Regs1}), {State1, <>, byte_size(I1) + byte_size(I2) + RelocJZOffset}; if_block_cond0(#state{regs = Regs0} = State0, {Reg, '&', Mask, '!=', Val}) when ?IS_UINT8_T(Mask) -> - Temp = first_avail(State0#state.available_regs), + Temp = first_avail(jit_regs:available_regs(Regs0)), I1 = jit_x86_64_asm:movq(Reg, Temp), I2 = jit_x86_64_asm:andb(Mask, Temp), I3 = jit_x86_64_asm:cmpb(Val, Temp), @@ -1125,21 +1070,14 @@ if_block_cond0(#state{regs = Regs0} = State0, {Reg, '&', Mask, '!=', Val}) when }. -spec if_block_free_reg(x86_64_register() | {free, x86_64_register()}, state()) -> state(). -if_block_free_reg({free, Reg}, #state{available_regs = AvR0, used_regs = UR0} = State0) -> +if_block_free_reg({free, Reg}, #state{regs = Regs0} = State0) -> Bit = reg_bit(Reg), State0#state{ - available_regs = AvR0 bor Bit, - used_regs = UR0 band (bnot Bit) + regs = jit_regs:free_reg(Regs0, Bit) }; if_block_free_reg(Reg, State0) when ?IS_GPR(Reg) -> State0. --spec merge_used_regs(state(), non_neg_integer()) -> state(). -merge_used_regs(#state{used_regs = UR} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - MergedAvail = ?AVAILABLE_REGS_MASK band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. - %%----------------------------------------------------------------------------- %% @doc Emit a shift register right by a fixed number of bits, effectively %% dividing it by 2^Shift @@ -1162,8 +1100,6 @@ shift_right( shift_right( #state{ stream_module = StreamModule, - available_regs = Avail, - used_regs = UR, stream = Stream0, regs = Regs0 } = State, @@ -1172,6 +1108,7 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I1 = jit_x86_64_asm:movq(Reg, ResultReg), @@ -1181,9 +1118,7 @@ shift_right( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ResultReg }. @@ -1209,8 +1144,6 @@ shift_right_arith( shift_right_arith( #state{ stream_module = StreamModule, - available_regs = Avail, - used_regs = UR, stream = Stream0, regs = Regs0 } = State, @@ -1219,6 +1152,7 @@ shift_right_arith( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I1 = jit_x86_64_asm:movq(Reg, ResultReg), @@ -1228,9 +1162,7 @@ shift_right_arith( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ResultReg }. @@ -1268,12 +1200,13 @@ call_func_ptr( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State0, FuncPtrTuple, Args ) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), + UsedRegs0 = jit_regs:used_regs(Regs0), FreeMask = lists:foldl( fun ({free, {ptr, Reg}}, Acc) -> Acc bor reg_bit(Reg); @@ -1354,9 +1287,7 @@ call_func_ptr( { State1#state{ stream = Stream9, - available_regs = AvailableRegs2, - used_regs = UsedRegs2, - regs = Regs1 + regs = jit_regs:set_masks(Regs1, AvailableRegs2, UsedRegs2) }, ResultReg }. @@ -1370,13 +1301,14 @@ set_args(State0, Args) -> set_args2(State0, Args, ParamRegs, ArgsRegs, ParamMask, ArgsMask). set_args2( - #state{stream = Stream0, stream_module = StreamModule, used_regs = UsedRegs} = State0, + #state{stream = Stream0, stream_module = StreamModule, regs = Regs0} = State0, Args, ParamRegs, ArgsRegs, ParamMask, ArgsMask ) -> + UsedRegs = jit_regs:used_regs(Regs0), AvailableScratchGP = ?SCRATCH_REGS_MASK band (bnot (ParamMask bor ArgsMask bor UsedRegs)), Offset = StreamModule:offset(Stream0), @@ -1400,8 +1332,11 @@ set_args2( ), State0#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), - used_regs = ParamMask bor NewUsedMask + regs = jit_regs:set_masks( + Regs0, + ?AVAILABLE_REGS_MASK band (bnot (ParamMask bor NewUsedMask)), + ParamMask bor NewUsedMask + ) }. parameter_regs(Args) -> @@ -1584,7 +1519,8 @@ move_to_vm_register_emit(State, 0, {ptr, Reg}) -> I1 = jit_x86_64_asm:andq(0, {0, Reg}), Stream1 = (State#state.stream_module):append(State#state.stream, I1), State#state{stream = Stream1}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, 0, {y_reg, Y}) -> +move_to_vm_register_emit(#state{regs = Regs0} = State, 0, {y_reg, Y}) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), I2 = jit_x86_64_asm:andq(0, {Y * 8, Temp}), @@ -1607,9 +1543,10 @@ move_to_vm_register_emit(State, N, {ptr, Reg}) when ?IS_SINT32_T(N) -> State#state.stream, jit_x86_64_asm:movq(N, {0, Reg}) ), State#state{stream = Stream1}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, N, {y_reg, Y}) when +move_to_vm_register_emit(#state{regs = Regs0} = State, N, {y_reg, Y}) when ?IS_SINT32_T(N) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), I2 = jit_x86_64_asm:movq(N, {Y * 8, Temp}), @@ -1617,9 +1554,10 @@ move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, N Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; % ?is_integer(Src), we need to use movabsq -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, N, {x_reg, X}) when +move_to_vm_register_emit(#state{regs = Regs0} = State, N, {x_reg, X}) when X < ?MAX_REG andalso is_integer(N) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(N, Temp), I2 = jit_x86_64_asm:movq(Temp, ?X_REG(X)), @@ -1627,28 +1565,31 @@ move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, N Regs1 = jit_regs:set_contents(Regs0, Temp, {imm, N}), State#state{stream = Stream1, regs = Regs1}; move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State, N, {x_reg, extra} + #state{regs = Regs0} = State, N, {x_reg, extra} ) when is_integer(N) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(N, Temp), I2 = jit_x86_64_asm:movq(Temp, ?X_REG(?MAX_REG)), Stream1 = (State#state.stream_module):append(State#state.stream, <>), Regs1 = jit_regs:set_contents(Regs0, Temp, {imm, N}), State#state{stream = Stream1, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, N, {ptr, Reg}) when +move_to_vm_register_emit(#state{regs = Regs0} = State, N, {ptr, Reg}) when is_integer(N) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(N, Temp), I2 = jit_x86_64_asm:movq(Temp, {0, Reg}), Stream1 = (State#state.stream_module):append(State#state.stream, <>), Regs1 = jit_regs:set_contents(Regs0, Temp, {imm, N}), State#state{stream = Stream1, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, N, {y_reg, Y}) when +move_to_vm_register_emit(#state{regs = Regs0} = State, N, {y_reg, Y}) when is_integer(N) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp1), @@ -1672,9 +1613,10 @@ move_to_vm_register_emit(State, Reg, {ptr, Dest}) when is_atom(Reg) -> I1 = jit_x86_64_asm:movq(Reg, {0, Dest}), Stream1 = (State#state.stream_module):append(State#state.stream, I1), State#state{stream = Stream1}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, Reg, {y_reg, Y}) when +move_to_vm_register_emit(#state{regs = Regs0} = State, Reg, {y_reg, Y}) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), I2 = jit_x86_64_asm:movq(Reg, {Y * 8, Temp}), @@ -1684,67 +1626,40 @@ move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State, R State#state{stream = Stream1, regs = Regs1}; % Src is x_reg, store in temporary register and call move_to_vm_register_emit for the four cases move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, {x_reg, X}, Dest + #state{regs = Regs0} = State0, {x_reg, X}, Dest ) when X < ?MAX_REG -> - Temp = first_avail(Avail), - TempBit = reg_bit(Temp), - I1 = jit_x86_64_asm:movq(?X_REG(X), Temp), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - Regs1 = jit_regs:set_contents(Regs0, Temp, {x_reg, X}), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = Avail band (bnot TempBit), regs = Regs1}, - Temp, - Dest - ), - State1#state{available_regs = Avail}; + with_temp(State0, Dest, fun(Temp) -> + {jit_x86_64_asm:movq(?X_REG(X), Temp), jit_regs:set_contents(Regs0, Temp, {x_reg, X})} + end); move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, {x_reg, extra}, Dest + #state{regs = Regs0} = State0, {x_reg, extra}, Dest ) -> - Temp = first_avail(Avail), - TempBit = reg_bit(Temp), - I1 = jit_x86_64_asm:movq(?X_REG(?MAX_REG), Temp), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - Regs1 = jit_regs:set_contents(Regs0, Temp, {x_reg, ?MAX_REG}), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = Avail band (bnot TempBit), regs = Regs1}, - Temp, - Dest - ), - State1#state{available_regs = Avail}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State0, {ptr, Reg}, Dest) -> - Temp = first_avail(Avail), - TempBit = reg_bit(Temp), - I1 = jit_x86_64_asm:movq({0, Reg}, Temp), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - Regs1 = jit_regs:invalidate_reg(Regs0, Temp), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = Avail band (bnot TempBit), regs = Regs1}, - Temp, - Dest - ), - State1#state{available_regs = Avail}; -move_to_vm_register_emit(#state{available_regs = Avail, regs = Regs0} = State0, {y_reg, Y}, Dest) -> - Temp = first_avail(Avail), - TempBit = reg_bit(Temp), - I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), - I2 = jit_x86_64_asm:movq({Y * 8, Temp}, Temp), - Stream1 = (State0#state.stream_module):append(State0#state.stream, <>), - Regs1 = jit_regs:set_contents(Regs0, Temp, {y_reg, Y}), - State1 = move_to_vm_register_emit( - State0#state{stream = Stream1, available_regs = Avail band (bnot TempBit), regs = Regs1}, - Temp, - Dest - ), - State1#state{available_regs = Avail}; + with_temp(State0, Dest, fun(Temp) -> + { + jit_x86_64_asm:movq(?X_REG(?MAX_REG), Temp), + jit_regs:set_contents(Regs0, Temp, {x_reg, ?MAX_REG}) + } + end); +move_to_vm_register_emit(#state{regs = Regs0} = State0, {ptr, Reg}, Dest) -> + with_temp(State0, Dest, fun(Temp) -> + {jit_x86_64_asm:movq({0, Reg}, Temp), jit_regs:invalidate_reg(Regs0, Temp)} + end); +move_to_vm_register_emit(#state{regs = Regs0} = State0, {y_reg, Y}, Dest) -> + with_temp(State0, Dest, fun(Temp) -> + I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), + I2 = jit_x86_64_asm:movq({Y * 8, Temp}, Temp), + {<>, jit_regs:set_contents(Regs0, Temp, {y_reg, Y})} + end); % term_to_float move_to_vm_register_emit( - #state{stream_module = StreamModule, available_regs = Avail, stream = Stream0, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {free, {ptr, Reg, 1}}, {fp_reg, F} ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq({8, Reg}, Reg), I2 = jit_x86_64_asm:movq(?FP_REGS, Temp), @@ -1755,6 +1670,29 @@ move_to_vm_register_emit( State1 = free_native_register(State0#state{regs = Regs1}, Reg), State1#state{stream = Stream1}. +-spec with_temp( + state(), + vm_register(), + fun((x86_64_register()) -> {binary(), jit_regs:regs()}) +) -> state(). +with_temp( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Dest, EmitFn +) -> + Avail = jit_regs:available_regs(Regs0), + Temp = first_avail(Avail), + TempBit = reg_bit(Temp), + {Code, Regs1} = EmitFn(Temp), + Stream1 = StreamModule:append(Stream0, Code), + State1 = move_to_vm_register_emit( + State0#state{ + stream = Stream1, + regs = jit_regs:set_available_regs(Regs1, Avail band (bnot TempBit)) + }, + Temp, + Dest + ), + State1#state{regs = jit_regs:set_available_regs(State1#state.regs, Avail)}. + %%----------------------------------------------------------------------------- %% @doc Emit a move of an array element (reg[x]) to a vm or a native register. %% @end @@ -1771,12 +1709,13 @@ move_to_vm_register_emit( Dest :: vm_register() | x86_64_register() ) -> state(). move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq({Index * 8, Reg}, Temp), I2 = jit_x86_64_asm:movq(Temp, ?X_REG(X)), @@ -1785,12 +1724,13 @@ move_array_element( Regs2 = jit_regs:set_contents(Regs1, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq({Index * 8, Reg}, Temp), I2 = jit_x86_64_asm:movq(Temp, {0, Dest}), @@ -1798,12 +1738,13 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {y_reg, Y} ) when is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp1), @@ -1826,8 +1767,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -1843,17 +1782,13 @@ move_array_element( Regs1 = jit_regs:invalidate_vm_loc(Regs0, {x_reg, X}), Regs2 = jit_regs:invalidate_reg(Regs1, IndexReg), State#state{ - available_regs = AvailableRegs0 bor IndexBit, - used_regs = UsedRegs0 band (bnot IndexBit), stream = Stream1, - regs = Regs2 + regs = jit_regs:free_reg(Regs2, IndexBit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -1868,23 +1803,20 @@ move_array_element( Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_reg(Regs0, IndexReg), State#state{ - available_regs = AvailableRegs0 bor IndexBit, - used_regs = UsedRegs0 band (bnot IndexBit), stream = Stream1, - regs = Regs1 + regs = jit_regs:free_reg(Regs1, IndexBit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, {free, IndexReg}, {y_reg, Y} ) when ?IS_GPR(IndexReg) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), Temp = first_avail(AvailableRegs0), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), I2 = jit_x86_64_asm:shlq(3, IndexReg), @@ -1899,10 +1831,8 @@ move_array_element( Regs2 = jit_regs:invalidate_reg(Regs1, Temp), Regs3 = jit_regs:invalidate_reg(Regs2, IndexReg), State#state{ - available_regs = AvailableRegs0 bor IndexBit, - used_regs = UsedRegs0 band (bnot IndexBit), stream = Stream1, - regs = Regs3 + regs = jit_regs:free_reg(Regs3, IndexBit) }. %%----------------------------------------------------------------------------- @@ -1936,13 +1866,12 @@ get_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, Index ) -> + Avail = jit_regs:available_regs(Regs0), ElemReg = first_avail(Avail), Bit = reg_bit(ElemReg), I1 = jit_x86_64_asm:movq({Index * 8, Reg}, ElemReg), @@ -1951,9 +1880,7 @@ get_array_element( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UsedRegs0 bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, ElemReg }. @@ -1975,12 +1902,13 @@ get_array_element( Index :: non_neg_integer() ) -> state(). move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {x_reg, X}, Reg, Index ) when X < ?MAX_REG andalso ?IS_GPR(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?X_REG(X), Temp), I2 = jit_x86_64_asm:movq(Temp, {Index * 8, Reg}), @@ -1988,12 +1916,13 @@ move_to_array_element( Regs1 = jit_regs:set_contents(Regs0, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {ptr, Source}, Reg, Index ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq({0, Source}, Temp), I2 = jit_x86_64_asm:movq(Temp, {Index * 8, Reg}), @@ -2001,12 +1930,13 @@ move_to_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y}, Reg, Index ) when ?IS_GPR(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), I2 = jit_x86_64_asm:movq({Y * 8, Temp}, Temp), @@ -2028,12 +1958,13 @@ move_to_array_element( Stream1 = StreamModule:append(Stream0, I1), State#state{stream = Stream1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Source, Reg, Index ) when is_integer(Source) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Source, Temp), I2 = jit_x86_64_asm:movq(Temp, {Index * 8, Reg}), @@ -2061,13 +1992,14 @@ move_to_array_element( ) when is_integer(Index) andalso is_integer(Offset) -> move_to_array_element(State, Source, BaseReg, Index + Offset); move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {x_reg, X}, BaseReg, IndexReg, Offset ) when X < ?MAX_REG andalso ?IS_GPR(BaseReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?X_REG(X), Temp), I2 = jit_x86_64_asm:movq(Temp, {Offset * ?WORD_SIZE, BaseReg, IndexReg, 8}), @@ -2075,13 +2007,14 @@ move_to_array_element( Regs1 = jit_regs:set_contents(Regs0, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y}, BaseReg, IndexReg, Offset ) when ?IS_GPR(BaseReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Temp), I2 = jit_x86_64_asm:movq({Y * 8, Temp}, Temp), @@ -2123,16 +2056,17 @@ move_to_native_register(#state{regs = Regs} = State, Value) -> case Contents =/= unknown andalso jit_regs:find_reg_with_contents(Regs, Contents) of {ok, CachedReg} -> Bit = reg_bit(CachedReg), - case State#state.used_regs band Bit of + Used = jit_regs:used_regs(Regs), + case Used band Bit of 0 -> - case State#state.available_regs band Bit of + Avail = jit_regs:available_regs(Regs), + case Avail band Bit of 0 -> move_to_native_register_emit(State, Value, Contents); _ -> { State#state{ - used_regs = State#state.used_regs bor Bit, - available_regs = State#state.available_regs band (bnot Bit) + regs = jit_regs:alloc_reg(Regs, Bit) }, CachedReg } @@ -2148,13 +2082,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, cp, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_x86_64_asm:movq(?CP, Reg), @@ -2163,9 +2096,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2183,8 +2114,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, Imm, @@ -2192,6 +2121,7 @@ move_to_native_register_emit( ) when is_integer(Imm) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = @@ -2206,9 +2136,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2216,13 +2144,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, extra}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_x86_64_asm:movq(?X_REG(?MAX_REG), Reg), @@ -2231,9 +2158,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2241,8 +2166,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, X}, @@ -2250,6 +2173,7 @@ move_to_native_register_emit( ) when X < ?MAX_REG -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_x86_64_asm:movq(?X_REG(X), Reg), @@ -2258,9 +2182,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor Bit, - available_regs = Avail band (bnot Bit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }; @@ -2268,13 +2190,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {y_reg, Y}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_x86_64_asm:movq(?Y_REGS, Reg), @@ -2285,9 +2206,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }. @@ -2321,12 +2240,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, Reg ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), Bit = reg_bit(SaveReg), I1 = jit_x86_64_asm:movq(Reg, SaveReg), @@ -2336,9 +2254,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, SaveReg }; @@ -2346,12 +2262,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {ptr, Reg} ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), Bit = reg_bit(SaveReg), I1 = jit_x86_64_asm:movq({0, Reg}, SaveReg), @@ -2360,9 +2275,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = Used bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, SaveReg }; @@ -2370,10 +2283,11 @@ copy_to_native_register(State, Reg) -> move_to_native_register(State, Reg). move_to_cp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y} ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Reg), I2 = jit_x86_64_asm:movq({Y * 8, Reg}, Reg), @@ -2384,10 +2298,11 @@ move_to_cp( State#state{stream = Stream1, regs = Regs1}. increment_sp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Offset ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), I1 = jit_x86_64_asm:movq(?Y_REGS, Reg), I2 = jit_x86_64_asm:addq(Offset * 8, Reg), @@ -2401,13 +2316,13 @@ set_continuation_to_label( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, labels = Labels, regs = Regs0 } = State, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Offset = StreamModule:offset(Stream0), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), @@ -2440,11 +2355,11 @@ set_continuation_to_offset( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), OffsetRef = make_ref(), Offset = StreamModule:offset(Stream0), @@ -2473,11 +2388,10 @@ get_module_index( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), Bit = reg_bit(Reg), I1 = jit_x86_64_asm:movq(?JITSTATE_MODULE, Reg), @@ -2488,9 +2402,7 @@ get_module_index( { State#state{ stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UsedRegs0 bor Bit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, Bit) }, Reg }. @@ -2507,13 +2419,14 @@ and_( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {State#state{stream = Stream1, regs = Regs1}, Reg}; and_( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Val ) when ?IS_GPR(Reg), is_integer(Val), Val < -16#80 orelse Val > 16#FFFFFFFF -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Val, TempReg), I2 = jit_x86_64_asm:andq(TempReg, Reg), @@ -2538,8 +2451,6 @@ and_( and_( #state{ stream_module = StreamModule, - available_regs = Avail, - used_regs = UR, stream = Stream0, regs = Regs0 } = State, @@ -2548,26 +2459,18 @@ and_( ) when ?IS_GPR(Reg), is_integer(Val), Val < -16#80 orelse Val > 16#FFFFFFFF -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I1 = jit_x86_64_asm:movabsq(Val, ResultReg), I2 = jit_x86_64_asm:andq(Reg, ResultReg), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), - { - State#state{ - stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, - regs = Regs1 - }, - ResultReg - }; + Regs2 = jit_regs:alloc_reg(Regs1, Bit), + {State#state{stream = Stream1, regs = Regs2}, ResultReg}; and_( #state{ stream_module = StreamModule, - available_regs = Avail, - used_regs = UR, stream = Stream0, regs = Regs0 } = State, @@ -2576,6 +2479,7 @@ and_( ) when ?IS_GPR(Reg) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), Bit = reg_bit(ResultReg), I1 = jit_x86_64_asm:movq(Reg, ResultReg), @@ -2586,15 +2490,8 @@ and_( end, Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_reg(Regs0, ResultReg), - { - State#state{ - stream = Stream1, - available_regs = Avail band (bnot Bit), - used_regs = UR bor Bit, - regs = Regs1 - }, - ResultReg - }. + Regs2 = jit_regs:alloc_reg(Regs1, Bit), + {State#state{stream = Stream1, regs = Regs2}, ResultReg}. or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, SrcReg) when is_atom(SrcReg) @@ -2604,11 +2501,11 @@ or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State#state{stream = Stream1, regs = Regs1}; or_( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Val ) when is_integer(Val), Val < -16#80000000 orelse Val > 16#7FFFFFFF -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Val, TempReg), I2 = jit_x86_64_asm:orq(TempReg, Reg), @@ -2629,11 +2526,11 @@ xor_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = Stat Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State#state{stream = Stream1, regs = Regs1}; xor_( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Val ) when is_integer(Val), Val < -16#80000000 orelse Val > 16#7FFFFFFF -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Val, TempReg), I2 = jit_x86_64_asm:xorq(TempReg, Reg), @@ -2650,12 +2547,12 @@ add( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, regs = Regs0 } = State, Reg, Val ) when is_integer(Val), Val < -16#80000000 orelse Val > 16#7FFFFFFF -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Val, TempReg), I2 = jit_x86_64_asm:addq(TempReg, Reg), @@ -2672,12 +2569,12 @@ sub( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, regs = Regs0 } = State, Reg, Val ) when is_integer(Val), Val < -16#80000000 orelse Val > 16#7FFFFFFF -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Val, TempReg), I2 = jit_x86_64_asm:subq(TempReg, Reg), @@ -2707,11 +2604,12 @@ mul(State, Reg, 64) -> shift_left(State, Reg, 6); mul( #state{ - stream_module = StreamModule, stream = Stream0, regs = Regs0, available_regs = Avail + stream_module = StreamModule, stream = Stream0, regs = Regs0 } = State, Reg, Val ) when is_integer(Val), (Val < -16#80000000 orelse Val > 16#7FFFFFFF) -> + Avail = jit_regs:available_regs(Regs0), TempReg = first_avail(Avail), I1 = jit_x86_64_asm:movabsq(Val, TempReg), I2 = jit_x86_64_asm:imulq(TempReg, Reg), @@ -2738,11 +2636,11 @@ mul( %% rdx is the native interface pointer and must be saved/restored. -spec div_(state(), x86_64_register(), x86_64_register()) -> {state(), rax}. div_( - #state{stream_module = StreamModule, stream = Stream0, regs = Regs0, available_regs = Avail} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DividendReg, DivisorReg ) -> + Avail = jit_regs:available_regs(Regs0), %% DivisorReg must not be rax (clobbered by dividend move) or rdx (clobbered by cqo). %% If DivisorReg is rax, move it to a temp register first. {I0, ActualDivisor, Regs1} = @@ -2775,11 +2673,11 @@ div_( %% rdx is the native interface pointer and must be saved/restored. -spec rem_(state(), x86_64_register(), x86_64_register()) -> {state(), x86_64_register()}. rem_( - #state{stream_module = StreamModule, stream = Stream0, regs = Regs0, available_regs = Avail} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DividendReg, DivisorReg ) -> + Avail = jit_regs:available_regs(Regs0), %% We need a temp register to save the remainder (rdx) before restoring rdx. %% This temp must not be rax (quotient) or the DivisorReg. RemTemp = first_avail( @@ -2816,21 +2714,14 @@ rem_( RemBit = reg_bit(RemTemp), Regs2 = jit_regs:invalidate_reg(Regs1, rax), Regs3 = jit_regs:invalidate_reg(Regs2, RemTemp), - { - State#state{ - stream = Stream1, - regs = Regs3, - available_regs = Avail band (bnot RemBit), - used_regs = State#state.used_regs bor RemBit - }, - RemTemp - }. + Regs4 = jit_regs:alloc_reg(Regs3, RemBit), + {State#state{stream = Stream1, regs = Regs4}, RemTemp}. -spec decrement_reductions_and_maybe_schedule_next(state()) -> state(). decrement_reductions_and_maybe_schedule_next( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State0 + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0 ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), Offset = StreamModule:offset(Stream0), @@ -2853,9 +2744,9 @@ decrement_reductions_and_maybe_schedule_next( (NewOffset - Offset - byte_size(I1) - byte_size(I2) - byte_size(I3)):32/little >> ), - State3 = merge_used_regs(State2#state{stream = Stream4}, State1#state.used_regs), - %% schedule_next clobbers caller-saved regs; invalidate cache at continuation. - State3#state{regs = jit_regs:invalidate_all(State1#state.regs)}. + %% schedule_next clobbers caller-saved regs; restore the pre-call masks + %% (preserved by invalidate_all) and drop any cached contents. + State2#state{stream = Stream4, regs = jit_regs:invalidate_all(State1#state.regs)}. -spec call_or_schedule_next(state(), non_neg_integer()) -> state(). call_or_schedule_next(State0, Label) -> @@ -2989,59 +2880,6 @@ reg_bit(r9) -> ?REG_BIT_R9; reg_bit(r10) -> ?REG_BIT_R10; reg_bit(r11) -> ?REG_BIT_R11. -first_avail(Mask) when Mask band ?REG_BIT_RAX =/= 0 -> rax; -first_avail(Mask) when Mask band ?REG_BIT_R11 =/= 0 -> r11; -first_avail(Mask) when Mask band ?REG_BIT_R10 =/= 0 -> r10; -first_avail(Mask) when Mask band ?REG_BIT_R9 =/= 0 -> r9; -first_avail(Mask) when Mask band ?REG_BIT_R8 =/= 0 -> r8; -first_avail(Mask) when Mask band ?REG_BIT_RCX =/= 0 -> rcx. - -%% Convert bitmask to list in reverse allocation order -%% Iteration order: rcx, r8, r9, r10, r11, rax -mask_to_list(0) -> []; -mask_to_list(Mask) -> mask_to_list_rcx(Mask). - -mask_to_list_rcx(Mask) when Mask band ?REG_BIT_RCX =/= 0 -> [rcx | mask_to_list_r8(Mask)]; -mask_to_list_rcx(Mask) -> mask_to_list_r8(Mask). - -mask_to_list_r8(Mask) when Mask band ?REG_BIT_R8 =/= 0 -> [r8 | mask_to_list_r9(Mask)]; -mask_to_list_r8(Mask) -> mask_to_list_r9(Mask). - -mask_to_list_r9(Mask) when Mask band ?REG_BIT_R9 =/= 0 -> [r9 | mask_to_list_r10(Mask)]; -mask_to_list_r9(Mask) -> mask_to_list_r10(Mask). - -mask_to_list_r10(Mask) when Mask band ?REG_BIT_R10 =/= 0 -> [r10 | mask_to_list_r11(Mask)]; -mask_to_list_r10(Mask) -> mask_to_list_r11(Mask). - -mask_to_list_r11(Mask) when Mask band ?REG_BIT_R11 =/= 0 -> [r11 | mask_to_list_rax(Mask)]; -mask_to_list_r11(Mask) -> mask_to_list_rax(Mask). - -mask_to_list_rax(Mask) when Mask band ?REG_BIT_RAX =/= 0 -> [rax]; -mask_to_list_rax(_Mask) -> []. - -args_regs(Args) -> - lists:map( - fun - ({free, {ptr, Reg}}) -> Reg; - ({free, Reg}) when is_atom(Reg) -> Reg; - ({free, Imm}) when is_integer(Imm) -> imm; - (offset) -> imm; - (ctx) -> ?CTX_REG; - (jit_state) -> ?JITSTATE_REG; - (Reg) when is_atom(Reg) -> Reg; - (Imm) when is_integer(Imm) -> imm; - ({ptr, Reg}) -> Reg; - ({x_reg, _}) -> ?CTX_REG; - ({y_reg, _}) -> ?CTX_REG; - ({fp_reg, _}) -> ?CTX_REG; - ({free, {x_reg, _}}) -> ?CTX_REG; - ({free, {y_reg, _}}) -> ?CTX_REG; - ({free, {fp_reg, _}}) -> ?CTX_REG; - ({avm_int64_t, _}) -> imm - end, - Args - ). - %%----------------------------------------------------------------------------- %% @doc Add a label at the current offset %% @end diff --git a/libs/jit/src/jit_xtensa.erl b/libs/jit/src/jit_xtensa.erl index 5ece009636..b0e6bd9ab9 100644 --- a/libs/jit/src/jit_xtensa.erl +++ b/libs/jit/src/jit_xtensa.erl @@ -60,6 +60,9 @@ set_continuation_to_offset/1, continuation_entry_point/1, get_module_index/1, + get_module/1, + get_cp_module/1, + get_cp_offset/1, and_/3, or_/3, add/3, @@ -166,8 +169,6 @@ offset :: non_neg_integer(), branches :: #{integer() | reference() => [{non_neg_integer(), non_neg_integer()}]}, jump_table_start :: non_neg_integer(), - available_regs :: non_neg_integer(), - used_regs :: non_neg_integer(), labels :: #{integer() | reference() => integer()}, variant :: non_neg_integer(), regs :: jit_regs:regs() @@ -204,9 +205,10 @@ -define(Y_REGS, {?CTX_REG, 16#14}). -define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}). -define(CP, {?CTX_REG, 16#5C}). --define(FP_REGS, {?CTX_REG, 16#60}). --define(BS, {?CTX_REG, 16#64}). --define(BS_OFFSET, {?CTX_REG, 16#68}). +-define(CP_MODULE, {?CTX_REG, 16#60}). +-define(FP_REGS, {?CTX_REG, 16#64}). +-define(BS, {?CTX_REG, 16#68}). +-define(BS_OFFSET, {?CTX_REG, 16#6C}). -define(JITSTATE_REG, a3). -define(JITSTATE_MODULE_OFFSET, 0). -define(JITSTATE_CONTINUATION_OFFSET, 16#4). @@ -313,11 +315,9 @@ new(Variant, StreamModule, Stream) -> branches = #{}, jump_table_start = 0, offset = StreamModule:offset(Stream), - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, labels = #{}, variant = Variant, - regs = jit_regs:new() + regs = jit_regs:new(?AVAILABLE_REGS_MASK, 0) }. %%----------------------------------------------------------------------------- @@ -363,67 +363,15 @@ debugger(#state{stream_module = StreamModule, stream = Stream0} = State) -> Stream1 = StreamModule:append(Stream0, jit_xtensa_asm:break(1, 15)), State#state{stream = Stream1}. -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently used native registers. This is used for -%% debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of used registers -%%----------------------------------------------------------------------------- --spec used_regs(state()) -> [xtensa_register()]. -used_regs(#state{used_regs = Used}) -> mask_to_list(Used). - -%%----------------------------------------------------------------------------- -%% @doc Return the list of currently available native scratch registers. This -%% is used for debugging and not in production. -%% @end -%% @param State current backend state -%% @return The list of available registers -%%----------------------------------------------------------------------------- --spec available_regs(state()) -> [xtensa_register()]. -available_regs(#state{available_regs = Available}) -> mask_to_list(Available). - -%%----------------------------------------------------------------------------- -%% @doc Free native registers. The passed list of registers can contain -%% registers, pointer to registers or other values that are ignored. -%% @end -%% @param State current backend state -%% @param Regs list of registers or other values -%% @return The updated backend state -%%----------------------------------------------------------------------------- --spec free_native_registers(state(), [value()]) -> state(). -free_native_registers(State, []) -> - State; -free_native_registers(State, [Reg | Rest]) -> - State1 = free_native_register(State, Reg), - free_native_registers(State1, Rest). - --spec free_native_register(state(), value()) -> state(). -free_native_register( - #state{available_regs = Available0, used_regs = Used0} = State, - Reg -) when is_atom(Reg) -> - Bit = reg_bit(Reg), - State#state{ - available_regs = Available0 bor Bit, used_regs = Used0 band (bnot Bit) - }; -free_native_register(State, {ptr, Reg}) -> - free_native_register(State, Reg); -free_native_register(State, _Other) -> - State. - -%%----------------------------------------------------------------------------- -%% @doc Assert that all native scratch registers are available. This is used -%% for debugging and not in production. -%% @end -%% @param State current backend state -%% @return ok -%%----------------------------------------------------------------------------- --spec assert_all_native_free(state()) -> ok. -assert_all_native_free(State) -> - 0 = State#state.used_regs, - ?AVAILABLE_REGS_MASK = State#state.available_regs, - ok. +%% Native-register allocation bookkeeping (used_regs/1, available_regs/1, +%% free_native_registers/2, free_native_register/2, assert_all_native_free/1, +%% first_avail/1, mask_to_list/1, args_regs/1, prepare_call_scratch/1) is shared +%% across the register-based backends and flows through jit_regs. +%% High registers (a15-a9) are clobbered by CALLX8 so prefer them to minimize saves. +-define(FIRST_AVAIL_REGS, [a15, a14, a13, a12, a11, a10, a9, a8, a7, a6, a5]). +-define(MASK_TO_LIST_REGS, [a15, a14, a13, a12, a11, a10, a9, a8, a7, a6, a5, a4, a3, a2, a1, a0]). +-define(JITSTATE_ARG_REG, jit_state). +-include("jit_backend_regs_impl.hrl"). %%----------------------------------------------------------------------------- %% @doc Emit the jump table at the beginning of the module. Branches will be @@ -619,28 +567,26 @@ call_primitive( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, - used_regs = Used + regs = Regs0 } = State, Primitive, Args -) when Available =/= 0 -> - TempReg = first_avail(Available), - TempBit = reg_bit(TempReg), - PrepCall = load_primitive_ptr(Primitive, TempReg), - Stream1 = StreamModule:append(Stream0, PrepCall), - StateCall = State#state{ - stream = Stream1, - available_regs = Available band (bnot TempBit), - used_regs = Used bor TempBit - }, - call_func_ptr(StateCall, {free, TempReg}, Args); -call_primitive( - #state{available_regs = 0} = State, - Primitive, - Args ) -> - call_func_ptr(State, {primitive, Primitive}, Args). + Available = jit_regs:available_regs(Regs0), + case Available of + 0 -> + call_func_ptr(State, {primitive, Primitive}, Args); + _ -> + TempReg = first_avail(Available), + TempBit = reg_bit(TempReg), + PrepCall = load_primitive_ptr(Primitive, TempReg), + Stream1 = StreamModule:append(Stream0, PrepCall), + StateCall = State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs0, TempBit) + }, + call_func_ptr(StateCall, {free, TempReg}, Args) + end. %%----------------------------------------------------------------------------- %% @doc Emit a jump (call without return) to a primitive with arguments. This @@ -675,20 +621,14 @@ call_primitive_last( ) -> %% Xtensa windowed ABI: CALLX8 to the primitive, move its return value %% from a10 (our view of callee's a2) into our a2, then RETW to C. - ParamRegs = lists:sublist(?PARAMETER_REGS, length(Args)), - ArgsRegs = args_regs(Args), - ArgsRegsMask = jit_regs:regs_to_mask(ArgsRegs, fun reg_bit/1), - ParamMask = jit_regs:regs_to_mask(ParamRegs, fun reg_bit/1), - ScratchMask = ?AVAILABLE_REGS_MASK band (bnot (ArgsRegsMask bor ParamMask)), - Temp = first_avail(ScratchMask), - TempBit = reg_bit(Temp), - AvailableRegs1 = ScratchMask band (bnot TempBit), - UsedMask = ?AVAILABLE_REGS_MASK band (bnot AvailableRegs1), + #{temp := Temp, available_mask := AvailableRegs1, used_mask := UsedMask} = + prepare_call_scratch(Args), PrepCall = load_primitive_ptr(Primitive, Temp), Stream1 = StreamModule:append(Stream0, PrepCall), State1 = State0#state{ - stream = Stream1, available_regs = AvailableRegs1, used_regs = UsedMask + stream = Stream1, + regs = jit_regs:set_masks(State0#state.regs, AvailableRegs1, UsedMask) }, Args1 = lists:map( @@ -712,9 +652,9 @@ call_primitive_last( Stream3 = StreamModule:append(Stream2, <>), State2#state{ stream = Stream3, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State2#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(State2#state.regs), ?AVAILABLE_REGS_MASK, 0 + ) } end. @@ -731,8 +671,7 @@ return_if_not_equal_to_ctx( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State, {free, Reg} ) -> @@ -753,9 +692,7 @@ return_if_not_equal_to_ctx( RegBit = reg_bit(Reg), State#state{ stream = Stream1, - available_regs = AvailableRegs0 bor RegBit, - used_regs = UsedRegs0 band (bnot RegBit), - regs = State#state.regs + regs = jit_regs:free_reg(Regs0, RegBit) }. %%----------------------------------------------------------------------------- @@ -800,11 +737,12 @@ jump_to_continuation( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Available, + regs = Regs0, offset = BaseOffset } = State0, {free, OffsetReg} ) -> + Available = jit_regs:available_regs(Regs0), Temp = first_avail(Available), % Calculate absolute address: native_code_base + target_offset % where native_code_base = current_pc + (BaseOffset - CurrentStreamOffset) @@ -825,9 +763,9 @@ jump_to_continuation( % Free all registers since this is a tail jump State0#state{ stream = Stream1, - available_regs = ?AVAILABLE_REGS_MASK, - used_regs = 0, - regs = jit_regs:unreachable(State0#state.regs) + regs = jit_regs:set_masks( + jit_regs:unreachable(Regs0), ?AVAILABLE_REGS_MASK, 0 + ) }. branch_to_offset_code(State, Offset, TargetOffset) -> @@ -840,7 +778,8 @@ branch_to_offset_code(State, Offset, TargetOffset) -> %% Far jump: use code_base + code-relative offset via indirect jump. %% Need two scratch regs: A8_REG (a8) for code_base/target, and one %% available register for the offset immediate. - #state{jump_table_start = JumpTableStart, available_regs = Avail} = State, + #state{jump_table_start = JumpTableStart, regs = RegsBR} = State, + Avail = jit_regs:available_regs(RegsBR), CodeRelativeTarget = TargetOffset - JumpTableStart, Temp = first_avail(Avail), I1 = jit_xtensa_asm:l32i(?A8_REG, ?JITSTATE_REG, ?JITSTATE_CODE_BASE_OFFSET), @@ -854,11 +793,12 @@ branch_to_label_code(State, Offset, Label, {Label, LabelOffset}) -> CodeBlock = branch_to_offset_code(State, Offset, LabelOffset), {State, CodeBlock}; branch_to_label_code( - #state{branches = Branches, available_regs = Avail, jump_table_start = JTS} = State0, + #state{branches = Branches, regs = RegsBL, jump_table_start = JTS} = State0, Offset, Label, false ) -> + Avail = jit_regs:available_regs(RegsBL), %% Reserve 24 bytes for forward branch placeholder. %% Near targets will use J + NOPs, far targets use indirect jump. %% Pick a temp register now and encode it in the relocation. @@ -904,9 +844,10 @@ if_block( Stream2, Replacements ), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}; + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}; if_block( #state{stream_module = StreamModule, stream = Stream0} = State0, Cond, @@ -922,9 +863,10 @@ if_block( JumpRel = OffsetAfter - JumpOffset - 4, NewJumpInstr = jit_xtensa_asm:j(JumpRel), Stream3 = StreamModule:replace(Stream2, JumpOffset, NewJumpInstr), - State3 = merge_used_regs(State2#state{stream = Stream3}, State1#state.used_regs), - MergedRegs = jit_regs:merge(State1#state.regs, State2#state.regs), - State3#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State1#state.regs, State2#state.regs, ?AVAILABLE_REGS_MASK + ), + State2#state{stream = Stream3, regs = MergedRegs}. %%----------------------------------------------------------------------------- %% @doc Emit an if else block, i.e. emit a test of a condition and @@ -963,8 +905,11 @@ if_else_block( %% Build the else block StateElse = State2#state{ stream = Stream4, - used_regs = State1#state.used_regs, - available_regs = State1#state.available_regs + regs = jit_regs:set_masks( + State2#state.regs, + jit_regs:available_regs(State1#state.regs), + jit_regs:used_regs(State1#state.regs) + ) }, State3 = BlockFalseFn(StateElse), Stream5 = State3#state.stream, @@ -974,9 +919,10 @@ if_else_block( NewElseJumpInstr = jit_xtensa_asm:j(FinalJumpOffset), 3 = byte_size(NewElseJumpInstr), Stream6 = StreamModule:replace(Stream5, ElseJumpOffset, NewElseJumpInstr), - State4 = merge_used_regs(State3#state{stream = Stream6}, State2#state.used_regs), - MergedRegs = jit_regs:merge(State2#state.regs, State3#state.regs), - State4#state{regs = MergedRegs}. + MergedRegs = jit_regs:merge( + State2#state.regs, State3#state.regs, ?AVAILABLE_REGS_MASK + ), + State3#state{stream = Stream6, regs = MergedRegs}. -spec if_block_cond(state(), condition()) -> {state(), non_neg_integer()}. @@ -1021,10 +967,11 @@ if_block_cond( RegOrTuple -> RegOrTuple end, %% Xtensa: load Val, blt Reg, Temp, +2; J placeholder + Available0 = jit_regs:available_regs(State0#state.regs), Temp = - case State0#state.available_regs of + case Available0 of 0 -> ?A8_REG; - _ -> first_avail(State0#state.available_regs) + _ -> first_avail(Available0) end, OffsetBefore = StreamModule:offset(Stream0), State1 = mov_immediate(State0, Temp, Val), @@ -1038,9 +985,10 @@ if_block_cond( JumpDelta = MovSize + byte_size(I1), {State3, JumpDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '<', Val} ) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = case Available of 0 -> ?A8_REG; @@ -1064,9 +1012,10 @@ if_block_cond( JumpDelta = MovSize + byte_size(I1), {State3, JumpDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {Val, '<', RegOrTuple} ) when is_integer(Val), Val >= 0, Val =< 255 -> + Available = jit_regs:available_regs(Regs0), Temp = case Available of 0 -> ?A8_REG; @@ -1090,9 +1039,10 @@ if_block_cond( JumpDelta = MovSize + byte_size(I1), {State3, JumpDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {Val, '<', RegOrTuple} ) when is_integer(Val) -> + Available = jit_regs:available_regs(Regs0), Temp = case Available of 0 -> ?A8_REG; @@ -1199,9 +1149,10 @@ if_block_cond( State2 = State1#state{stream = Stream1}, {State2, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Available} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '!=', Val} ) when is_integer(Val) andalso Val >= 0 andalso Val =< 255 -> + Available = jit_regs:available_regs(Regs0), Temp = case Available of 0 -> ?A8_REG; @@ -1260,9 +1211,10 @@ if_block_cond( State2 = State1#state{stream = Stream1}, {State2, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '==', Val} ) when is_integer(Val) andalso Val >= 0 andalso Val =< 255 -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1298,9 +1250,10 @@ if_block_cond( State3 = if_block_free_reg({free, RegB}, State2), {State3, byte_size(I1)}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '==', Val} ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1324,9 +1277,10 @@ if_block_cond( JumpDelta = MovSize + byte_size(I1), {State3, JumpDelta}; if_block_cond( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {RegOrTuple, '!=', Val} ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1395,11 +1349,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, regs = Regs0 } = State0, {RegOrTuple, '&', Val, '!=', 0} ) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1430,11 +1384,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, regs = Regs0 } = State0, {Reg, '&', 16#F, '!=', 16#F} ) when ?IS_GPR(Reg) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1473,10 +1427,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail + regs = Regs0 } = State0, {Reg, '&', Mask, '!=', Val} ) when ?IS_GPR(Reg), Val =/= 0 -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1487,7 +1442,11 @@ if_block_cond( I1 = jit_xtensa_asm:mov(Temp, Reg), Stream1 = StreamModule:append(Stream0, I1), State1 = State0#state{stream = Stream1}, - {State2, Temp} = and_(State1#state{available_regs = AT}, {free, Temp}, Mask), + {State2, Temp} = and_( + State1#state{regs = jit_regs:set_available_regs(State1#state.regs, AT)}, + {free, Temp}, + Mask + ), Stream2 = State2#state.stream, case Val of _ when ?IS_GPR(Val) -> @@ -1496,7 +1455,11 @@ if_block_cond( BranchDelta = StreamModule:offset(Stream2) - OffsetBefore + byte_size(I_beq), Stream3 = StreamModule:append(Stream2, <>), State3 = State2#state{ - stream = Stream3, available_regs = State2#state.available_regs bor reg_bit(Temp) + stream = Stream3, + regs = jit_regs:set_available_regs( + State2#state.regs, + jit_regs:available_regs(State2#state.regs) bor reg_bit(Temp) + ) }, {State3, BranchDelta}; _ -> @@ -1507,7 +1470,11 @@ if_block_cond( _ -> first_avail(AT) end, AT2 = AT band (bnot reg_bit(MaskReg)), - State3 = mov_immediate(State2#state{available_regs = AT2}, MaskReg, Val), + State3 = mov_immediate( + State2#state{regs = jit_regs:set_available_regs(State2#state.regs, AT2)}, + MaskReg, + Val + ), Stream3 = State3#state.stream, I_beq2 = jit_xtensa_asm:bne(Temp, MaskReg, 2), JPlaceholder3 = <<16#FF, 16#FF, 16#FF>>, @@ -1515,7 +1482,11 @@ if_block_cond( Stream4 = StreamModule:append(Stream3, <>), State4 = State3#state{ stream = Stream4, - available_regs = State3#state.available_regs bor reg_bit(Temp) bor reg_bit(MaskReg) + regs = jit_regs:set_available_regs( + State3#state.regs, + jit_regs:available_regs(State3#state.regs) bor reg_bit(Temp) bor + reg_bit(MaskReg) + ) }, {State4, BranchDelta} end; @@ -1523,10 +1494,11 @@ if_block_cond( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailRegs + regs = Regs0 } = State0, {{free, Reg} = RegTuple, '&', Mask, '!=', Val} ) when ?IS_GPR(Reg), Val =/= 0 -> + AvailRegs = jit_regs:available_regs(Regs0), OffsetBefore = StreamModule:offset(Stream0), {State1, Reg} = and_(State0, RegTuple, Mask), Stream1 = State1#state.stream, @@ -1541,38 +1513,35 @@ if_block_cond( {State3, BranchDelta}; _ -> %% Val is an immediate - need temp register - MaskReg = first_avail(State1#state.available_regs), - AT = State1#state.available_regs band (bnot reg_bit(MaskReg)), - State2 = mov_immediate(State1#state{available_regs = AT}, MaskReg, Val), + State1Avail = jit_regs:available_regs(State1#state.regs), + MaskReg = first_avail(State1Avail), + AT = State1Avail band (bnot reg_bit(MaskReg)), + State2 = mov_immediate( + State1#state{regs = jit_regs:set_available_regs(State1#state.regs, AT)}, + MaskReg, + Val + ), Stream2 = State2#state.stream, I_beq4 = jit_xtensa_asm:bne(Reg, MaskReg, 2), JPlaceholder6 = <<16#FF, 16#FF, 16#FF>>, BranchDelta = StreamModule:offset(Stream2) - OffsetBefore + byte_size(I_beq4), Stream3 = StreamModule:append(Stream2, <>), - State3 = State2#state{stream = Stream3, available_regs = AvailRegs}, + State3 = State2#state{ + stream = Stream3, + regs = jit_regs:set_available_regs(State2#state.regs, AvailRegs) + }, State4 = if_block_free_reg(RegTuple, State3), {State4, BranchDelta} end. -spec if_block_free_reg(xtensa_register() | {free, xtensa_register()}, state()) -> state(). if_block_free_reg({free, Reg}, State0) -> - #state{available_regs = AvR0, used_regs = UR0} = State0, + #state{regs = Regs0} = State0, Bit = reg_bit(Reg), - AvR1 = AvR0 bor Bit, - UR1 = UR0 band (bnot Bit), - State0#state{ - available_regs = AvR1, - used_regs = UR1 - }; + State0#state{regs = jit_regs:free_reg(Regs0, Bit)}; if_block_free_reg(Reg, State0) when ?IS_GPR(Reg) -> State0. --spec merge_used_regs(state(), non_neg_integer()) -> state(). -merge_used_regs(#state{used_regs = UR} = State, OtherUR) -> - MergedUR = UR bor OtherUR, - MergedAvail = ?AVAILABLE_REGS_MASK band (bnot MergedUR), - State#state{used_regs = MergedUR, available_regs = MergedAvail}. - %%----------------------------------------------------------------------------- %% @doc Emit a shift register right by a fixed number of bits, effectively %% dividing it by 2^Shift @@ -1593,13 +1562,13 @@ shift_right( Regs1 = jit_regs:invalidate_reg(Regs0, Reg), {State#state{stream = Stream1, regs = Regs1}, Reg}; shift_right( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Shift ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -1615,8 +1584,6 @@ shift_right( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1624,6 +1591,7 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) andalso Shift =< 15 -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I = jit_xtensa_asm:srli(ResultReg, Reg, Shift), @@ -1632,9 +1600,7 @@ shift_right( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }; @@ -1642,8 +1608,6 @@ shift_right( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1651,6 +1615,7 @@ shift_right( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I1 = jit_xtensa_asm:movi(ResultReg, Shift), @@ -1661,9 +1626,7 @@ shift_right( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -1701,8 +1664,6 @@ shift_right_arith( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UR, regs = Regs0 } = State, Reg, @@ -1710,6 +1671,7 @@ shift_right_arith( ) when ?IS_GPR(Reg) andalso is_integer(Shift) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = first_avail(Avail), ResultBit = reg_bit(ResultReg), I = jit_xtensa_asm:srai(ResultReg, Reg, Shift), @@ -1718,9 +1680,7 @@ shift_right_arith( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -1760,12 +1720,13 @@ call_func_ptr( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0Mask, - used_regs = UsedRegs0Mask + regs = Regs0 } = State0, FuncPtrTuple, Args ) -> + AvailableRegs0Mask = jit_regs:available_regs(Regs0), + UsedRegs0Mask = jit_regs:used_regs(Regs0), AvailableRegs0 = mask_to_list(AvailableRegs0Mask), UsedRegs0 = mask_to_list(UsedRegs0Mask), FreeRegs = lists:flatmap( @@ -1820,9 +1781,12 @@ call_func_ptr( %% safe to reuse as scratch among the currently-used registers. SetArgsRegsOnlyAvailableArgs = (RegsToSave -- RegArgsRegs) ++ AvailableRegs0, State1 = State0#state{ - available_regs = jit_regs:regs_to_mask(SetArgsRegsOnlyAvailableArgs, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask( - ?AVAILABLE_REGS -- SetArgsRegsOnlyAvailableArgs, fun reg_bit/1 + regs = jit_regs:set_masks( + Regs0, + jit_regs:regs_to_mask(SetArgsRegsOnlyAvailableArgs, fun reg_bit/1), + jit_regs:regs_to_mask( + ?AVAILABLE_REGS -- SetArgsRegsOnlyAvailableArgs, fun reg_bit/1 + ) ), stream = Stream1 }, @@ -1892,8 +1856,13 @@ call_func_ptr( end, State3 = State1#state{ - available_regs = jit_regs:regs_to_mask(SetArgsAvailableRegs, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask(?AVAILABLE_REGS -- SetArgsAvailableRegs, fun reg_bit/1), + regs = jit_regs:set_masks( + State1#state.regs, + jit_regs:regs_to_mask(SetArgsAvailableRegs, fun reg_bit/1), + jit_regs:regs_to_mask( + ?AVAILABLE_REGS -- SetArgsAvailableRegs, fun reg_bit/1 + ) + ), stream = Stream3 }, @@ -1943,9 +1912,11 @@ call_func_ptr( { State4#state{ stream = Stream8, - available_regs = jit_regs:regs_to_mask(AvailableRegs3, fun reg_bit/1), - used_regs = jit_regs:regs_to_mask(UsedRegs2, fun reg_bit/1), - regs = Regs1 + regs = jit_regs:set_masks( + Regs1, + jit_regs:regs_to_mask(AvailableRegs3, fun reg_bit/1), + jit_regs:regs_to_mask(UsedRegs2, fun reg_bit/1) + ) }, ResultReg }. @@ -1986,11 +1957,12 @@ pop_registers([], _AlignedStackBytes, _StreamModule, Stream0) -> Stream0. set_registers_args( - #state{used_regs = UsedRegsMask} = State0, + #state{regs = Regs0} = State0, Args, ParamRegs, StackOffset ) -> + UsedRegsMask = jit_regs:used_regs(Regs0), UsedRegs = mask_to_list(UsedRegsMask), ArgsRegs = args_regs(Args), AvailableScratchGP = ((?AVAILABLE_REGS -- ParamRegs) -- ArgsRegs) -- UsedRegs, @@ -2009,10 +1981,15 @@ set_registers_args( ), State1#state{ stream = Stream1, - available_regs = jit_regs:regs_to_mask( - ?AVAILABLE_REGS -- ParamRegs -- NewUsedRegs, fun reg_bit/1 - ), - used_regs = jit_regs:regs_to_mask(ParamRegs ++ (NewUsedRegs -- ParamRegs), fun reg_bit/1) + regs = jit_regs:set_masks( + State1#state.regs, + jit_regs:regs_to_mask( + ?AVAILABLE_REGS -- ParamRegs -- NewUsedRegs, fun reg_bit/1 + ), + jit_regs:regs_to_mask( + ParamRegs ++ (NewUsedRegs -- ParamRegs), fun reg_bit/1 + ) + ) }. parameter_regs(Args) -> @@ -2176,12 +2153,12 @@ set_registers_args1( Stream1 = StreamModule:append(Stream0, I), State#state{stream = Stream1}; set_registers_args1( - #state{stream_module = StreamModule, stream = Stream0, available_regs = AvailRegs, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, X}, Reg, _StackOffset ) -> + AvailRegs = jit_regs:available_regs(Regs0), Code = ldr_y_reg(Reg, X, AvailRegs), Stream1 = StreamModule:append(Stream0, Code), Regs1 = @@ -2244,10 +2221,11 @@ move_to_vm_register_emit(State0, Src, {ptr, Reg}) when is_atom(Src) -> Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), State0#state{stream = Stream1}; move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, Src, {y_reg, Y} + #state{regs = Regs0} = State0, Src, {y_reg, Y} ) when is_atom(Src) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = case Avail of 0 -> ?A8_REG; @@ -2265,10 +2243,11 @@ move_to_vm_register_emit( State0#state{stream = Stream1, regs = Regs2}; % Source is an integer to y_reg (optimized: ldr first, then movs) move_to_vm_register_emit( - #state{available_regs = Avail, regs = Regs0} = State0, N, {y_reg, Y} + #state{regs = Regs0} = State0, N, {y_reg, Y} ) when is_integer(N), N >= 0, N =< 255 -> + Avail = jit_regs:available_regs(Regs0), Temp1 = case Avail of 0 -> ?A8_REG; @@ -2292,97 +2271,84 @@ move_to_vm_register_emit( end, State0#state{stream = Stream1, regs = Regs1}; % Source is an integer (0-255 for movs, negative values need different handling) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N), N >= 0, N =< 255 -> - Temp = - case AR0 of - 0 -> ?A8_REG; - _ -> first_avail(AR0) + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = mov_immediate(Temp, N), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) end, - AT = AR0 band (bnot reg_bit(Temp)), - I1 = mov_immediate(Temp, N), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {imm, N}), - State1#state{available_regs = AR0, regs = Regs1}; + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); %% Handle large values using simple literal pool (branch-over pattern) -move_to_vm_register_emit(#state{available_regs = AR0} = State0, N, Dest) when +move_to_vm_register_emit(State0, N, Dest) when is_integer(N) -> - Temp = - case AR0 of - 0 -> ?A8_REG; - _ -> first_avail(AR0) + with_temp( + State0, + fun(StateT, Temp, _AT) -> + State1 = mov_immediate(StateT, Temp, N), + move_to_vm_register(State1, Temp, Dest) end, - AT = AR0 band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, N), - State2 = move_to_vm_register(State1, Temp, Dest), - Regs1 = jit_regs:set_contents(State2#state.regs, Temp, {imm, N}), - State2#state{available_regs = AR0, regs = Regs1}; + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {imm, N}) end + ); % Source is a VM register -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, extra}, Dest) -> - Temp = - case AR0 of - 0 -> ?A8_REG; - _ -> first_avail(AR0) - end, - AT = AR0 band (bnot reg_bit(Temp)), - {BaseReg, Off} = ?X_REG(?MAX_REG), - I1 = jit_xtensa_asm:l32i(Temp, BaseReg, Off), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, extra}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {x_reg, X}, Dest) -> - Temp = - case AR0 of - 0 -> ?A8_REG; - _ -> first_avail(AR0) +move_to_vm_register_emit(State0, {x_reg, extra}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + {BaseReg, Off} = ?X_REG(?MAX_REG), + I1 = jit_xtensa_asm:l32i(Temp, BaseReg, Off), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) end, - AT = AR0 band (bnot reg_bit(Temp)), - {XReg, X_REGOffset} = ?X_REG(X), - I1 = jit_xtensa_asm:l32i(Temp, XReg, X_REGOffset), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:set_contents(State1#state.regs, Temp, {x_reg, X}), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {ptr, Reg}, Dest) -> - Temp = - case AR0 of - 0 -> ?A8_REG; - _ -> first_avail(AR0) + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, extra}) end + ); +move_to_vm_register_emit(State0, {x_reg, X}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + {XReg, X_REGOffset} = ?X_REG(X), + I1 = jit_xtensa_asm:l32i(Temp, XReg, X_REGOffset), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) end, - AT = AR0 band (bnot reg_bit(Temp)), - I1 = jit_xtensa_asm:l32i(Temp, Reg, 0), - Stream1 = (State0#state.stream_module):append(State0#state.stream, I1), - State1 = move_to_vm_register(State0#state{stream = Stream1, available_regs = AT}, Temp, Dest), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Temp), - State1#state{available_regs = AR0, regs = Regs1}; -move_to_vm_register_emit(#state{available_regs = AR0} = State0, {y_reg, Y}, Dest) -> - Temp = - case AR0 of - 0 -> ?A8_REG; - _ -> first_avail(AR0) + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {x_reg, X}) end + ); +move_to_vm_register_emit(State0, {ptr, Reg}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, _AT) -> + I1 = jit_xtensa_asm:l32i(Temp, Reg, 0), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, I1), + move_to_vm_register(StateT#state{stream = Stream1}, Temp, Dest) end, - AT = AR0 band (bnot reg_bit(Temp)), - Code = ldr_y_reg(Temp, Y, AT), - Stream1 = (State0#state.stream_module):append(State0#state.stream, Code), - Regs1 = - case AT of - 0 -> State0#state.regs; - _ -> jit_regs:invalidate_reg(State0#state.regs, first_avail(AT)) + fun(Regs, Temp) -> jit_regs:invalidate_reg(Regs, Temp) end + ); +move_to_vm_register_emit(State0, {y_reg, Y}, Dest) -> + with_temp( + State0, + fun(StateT, Temp, AT) -> + Code = ldr_y_reg(Temp, Y, AT), + Stream1 = (StateT#state.stream_module):append(StateT#state.stream, Code), + % ldr_y_reg clobbers first_avail(AT) as a hidden temp for the Y_REGS pointer + Regs1 = + case AT of + 0 -> StateT#state.regs; + _ -> jit_regs:invalidate_reg(StateT#state.regs, first_avail(AT)) + end, + move_to_vm_register(StateT#state{stream = Stream1, regs = Regs1}, Temp, Dest) end, - State1 = move_to_vm_register( - State0#state{stream = Stream1, available_regs = AT, regs = Regs1}, Temp, Dest - ), - Regs2 = jit_regs:set_contents(State1#state.regs, Temp, {y_reg, Y}), - State1#state{available_regs = AR0, regs = Regs2}; + fun(Regs, Temp) -> jit_regs:set_contents(Regs, Temp, {y_reg, Y}) end + ); % term_to_float move_to_vm_register_emit( #state{ stream_module = StreamModule, - available_regs = Avail, + regs = Regs0, stream = Stream0, variant = Variant } = @@ -2390,6 +2356,7 @@ move_to_vm_register_emit( {free, {ptr, Reg, 1}}, {fp_reg, F} ) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Temp2 = first_avail(Avail band (bnot reg_bit(Temp1))), {BaseReg, Off} = ?FP_REGS, @@ -2412,6 +2379,24 @@ move_to_vm_register_emit( Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(State1#state.regs, Temp1), Temp2), State1#state{stream = Stream1, regs = Regs1}. +-spec with_temp( + state(), + fun((state(), xtensa_register(), non_neg_integer()) -> state()), + fun((jit_regs:regs(), xtensa_register()) -> jit_regs:regs()) +) -> state(). +with_temp(#state{regs = Regs0} = State0, EmitFun, ContentsFun) -> + AR0 = jit_regs:available_regs(Regs0), + %% When no register is free, fall back to a8 as the scratch (xtensa reserves it). + Temp = + case AR0 of + 0 -> ?A8_REG; + _ -> first_avail(AR0) + end, + AT = AR0 band (bnot reg_bit(Temp)), + State1 = EmitFun(State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, AT), + Regs1 = jit_regs:set_available_regs(ContentsFun(State1#state.regs, Temp), AR0), + State1#state{regs = Regs1}. + %%----------------------------------------------------------------------------- %% @doc Emit a move of an array element (reg[x]) to a vm or a native register. %% @end @@ -2428,12 +2413,12 @@ move_to_vm_register_emit( vm_register() | xtensa_register() ) -> state(). move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {x_reg, X} ) when X < ?MAX_REG andalso is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_xtensa_asm:l32i(Temp, Reg, Index * 4), {BaseReg, Off} = ?X_REG(X), @@ -2443,12 +2428,12 @@ move_array_element( Regs2 = jit_regs:set_contents(Regs1, Temp, {x_reg, X}), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {ptr, Dest} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_xtensa_asm:l32i(Temp, Reg, Index * 4), I2 = jit_xtensa_asm:s32i(Temp, Dest, 0), @@ -2456,12 +2441,12 @@ move_array_element( Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State#state{stream = Stream1, regs = Regs1}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Reg, Index, {y_reg, Y} ) when is_atom(Reg) andalso is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp1 = first_avail(Avail), Avail2 = Avail band (bnot reg_bit(Temp1)), Temp2 = first_avail(Avail2), @@ -2474,12 +2459,12 @@ move_array_element( Regs2 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs1, Temp1), Temp2), State#state{stream = Stream1, regs = Regs2}; move_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {free, Reg}, Index, {y_reg, Y} ) when is_integer(Index) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), AT = Avail band (bnot reg_bit(Temp)), I1 = jit_xtensa_asm:l32i(Reg, Reg, Index * 4), @@ -2499,8 +2484,6 @@ move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, @@ -2513,22 +2496,17 @@ move_array_element( {BaseReg, Off} = ?X_REG(X), I4 = jit_xtensa_asm:s32i(IndexReg, BaseReg, Off), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), Regs1 = jit_regs:invalidate_vm_loc(Regs0, {x_reg, X}), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs1 + regs = jit_regs:free_reg(Regs1, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0 + regs = Regs0 } = State, Reg, {free, IndexReg}, @@ -2539,26 +2517,22 @@ move_array_element( I3 = jit_xtensa_asm:l32i(IndexReg, IndexReg, 0), I4 = jit_xtensa_asm:s32i(IndexReg, PtrReg, 0), Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append(Stream0, <>), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, - stream = Stream1 + stream = Stream1, + regs = jit_regs:free_reg(Regs0, Bit) }; move_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = AvailableRegs0, - used_regs = UsedRegs0, regs = Regs0 } = State, Reg, {free, IndexReg}, {y_reg, Y} ) when is_atom(IndexReg) -> + AvailableRegs0 = jit_regs:available_regs(Regs0), Temp = first_avail(AvailableRegs0), AT = AvailableRegs0 band (bnot reg_bit(Temp)), I1 = jit_xtensa_asm:slli(IndexReg, IndexReg, 2), @@ -2567,18 +2541,14 @@ move_array_element( Code = str_y_reg(IndexReg, Y, Temp, AT), I4 = Code, Bit = reg_bit(IndexReg), - AvailableRegs1 = AvailableRegs0 bor Bit, - UsedRegs1 = UsedRegs0 band (bnot Bit), Stream1 = StreamModule:append( Stream0, <> ), Regs1a = jit_regs:invalidate_vm_loc(Regs0, {y_reg, Y}), Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs1a, IndexReg), Temp), State#state{ - available_regs = AvailableRegs1, - used_regs = UsedRegs1, stream = Stream1, - regs = Regs1 + regs = jit_regs:free_reg(Regs1, Bit) }. %% @doc move reg[x] to a vm or native register @@ -2601,12 +2571,12 @@ get_array_element( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0 + regs = Regs0 } = State, Reg, Index ) -> + Avail = jit_regs:available_regs(Regs0), ElemReg = case Avail of 0 -> ?A8_REG; @@ -2618,8 +2588,7 @@ get_array_element( { State#state{ stream = Stream1, - available_regs = Avail band (bnot ElemBit), - used_regs = UsedRegs0 bor ElemBit + regs = jit_regs:alloc_reg(Regs0, ElemBit) }, ElemReg }. @@ -2638,12 +2607,12 @@ move_to_array_element( Stream1 = StreamModule:append(Stream0, I1), State0#state{stream = Stream1}; move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, ValueReg, Reg, IndexReg ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(Reg) andalso ?IS_GPR(IndexReg) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -2675,13 +2644,13 @@ move_to_array_element( ) when is_integer(IndexReg) andalso is_integer(Offset) -> move_to_array_element(State, Value, BaseReg, IndexReg + Offset); move_to_array_element( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, ValueReg, BaseReg, IndexReg, Offset ) when ?IS_GPR(ValueReg) andalso ?IS_GPR(IndexReg) andalso is_integer(Offset) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -2702,10 +2671,11 @@ move_to_array_element( Offset ) -> {State1, ValueReg} = copy_to_native_register(State0, Value), + State1Avail = jit_regs:available_regs(State1#state.regs), Temp = - case State1#state.available_regs of + case State1Avail of 0 -> ?A8_REG; - _ -> first_avail(State1#state.available_regs) + _ -> first_avail(State1Avail) end, I1 = jit_xtensa_asm:addi(Temp, IndexReg, Offset), I2 = jit_xtensa_asm:slli(Temp, Temp, 2), @@ -2725,16 +2695,17 @@ move_to_native_register(#state{regs = Regs} = State, Value) -> case Contents =/= unknown andalso jit_regs:find_reg_with_contents(Regs, Contents) of {ok, CachedReg} -> Bit = reg_bit(CachedReg), - case State#state.used_regs band Bit of + Used0 = jit_regs:used_regs(Regs), + case Used0 band Bit of 0 -> - case State#state.available_regs band Bit of + Avail0 = jit_regs:available_regs(Regs), + case Avail0 band Bit of 0 -> move_to_native_register_emit(State, Value, Contents); _ -> { State#state{ - used_regs = State#state.used_regs bor Bit, - available_regs = State#state.available_regs band (bnot Bit) + regs = jit_regs:alloc_reg(Regs, Bit) }, CachedReg } @@ -2750,13 +2721,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, cp, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), {BaseReg, Off} = ?CP, @@ -2766,9 +2736,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2783,8 +2751,6 @@ move_to_native_register_emit( {State#state{stream = Stream1, regs = Regs1}, Reg}; move_to_native_register_emit( #state{ - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State0, Imm, @@ -2792,6 +2758,7 @@ move_to_native_register_emit( ) when is_integer(Imm) -> + Avail = jit_regs:available_regs(Regs0), Reg = case Avail of 0 -> ?A8_REG; @@ -2800,22 +2767,19 @@ move_to_native_register_emit( RegBit = reg_bit(Reg), Regs1 = jit_regs:set_contents(Regs0, Reg, Contents), State1 = State0#state{ - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, {move_to_native_register(State1, Imm, Reg), Reg}; move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, extra}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = case Avail of 0 -> ?A8_REG; @@ -2829,9 +2793,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2839,8 +2801,6 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {x_reg, X}, @@ -2848,6 +2808,7 @@ move_to_native_register_emit( ) when X < ?MAX_REG -> + Avail = jit_regs:available_regs(Regs0), Reg = case Avail of 0 -> ?A8_REG; @@ -2861,9 +2822,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - used_regs = Used bor RegBit, - available_regs = Avail band (bnot RegBit), - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2871,13 +2830,12 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {y_reg, Y}, Contents ) -> + Avail = jit_regs:available_regs(Regs0), Reg = case Avail of 0 -> ?A8_REG; @@ -2896,9 +2854,7 @@ move_to_native_register_emit( { State#state{ stream = Stream1, - available_regs = AvailT, - used_regs = Used bor RegBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }; @@ -2906,18 +2862,17 @@ move_to_native_register_emit( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used + regs = Regs0 } = State, {fp_reg, F}, _Contents ) -> + Avail = jit_regs:available_regs(Regs0), RegA = first_avail(Avail), RegABit = reg_bit(RegA), Avail2 = Avail band (bnot RegABit), RegB = first_avail(Avail2), RegBBit = reg_bit(RegB), - AvailT = Avail2 band (bnot RegBBit), {BaseReg, Off} = ?FP_REGS, I1 = jit_xtensa_asm:l32i(RegB, BaseReg, Off), I2 = jit_xtensa_asm:l32i(RegA, RegB, F * 8), @@ -2926,7 +2881,8 @@ move_to_native_register_emit( Stream1 = StreamModule:append(Stream0, Code), { State#state{ - stream = Stream1, available_regs = AvailT, used_regs = Used bor RegABit bor RegBBit + stream = Stream1, + regs = jit_regs:alloc_reg(Regs0, RegABit bor RegBBit) }, {fp, RegA, RegB} }. @@ -2973,11 +2929,11 @@ move_to_native_register( Regs1 = jit_regs:set_contents(Regs0, RegDst, {x_reg, X}), State#state{stream = Stream1, regs = Regs1}; move_to_native_register( - #state{stream_module = StreamModule, stream = Stream0, available_regs = AT, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y}, RegDst ) -> + AT = jit_regs:available_regs(Regs0), Code = ldr_y_reg(RegDst, Y, AT), Stream1 = StreamModule:append(Stream0, Code), Regs1 = @@ -3008,12 +2964,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, Reg ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = jit_xtensa_asm:mov(SaveReg, Reg), @@ -3023,9 +2978,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, SaveBit) }, SaveReg }; @@ -3033,12 +2986,11 @@ copy_to_native_register( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = Used, regs = Regs0 } = State, {ptr, Reg} ) when is_atom(Reg) -> + Avail = jit_regs:available_regs(Regs0), SaveReg = first_avail(Avail), SaveBit = reg_bit(SaveReg), I1 = jit_xtensa_asm:l32i(SaveReg, Reg, 0), @@ -3047,9 +2999,7 @@ copy_to_native_register( { State#state{ stream = Stream1, - available_regs = Avail band (bnot SaveBit), - used_regs = Used bor SaveBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, SaveBit) }, SaveReg }; @@ -3057,16 +3007,21 @@ copy_to_native_register(State, Reg) -> move_to_native_register(State, Reg). move_to_cp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, {y_reg, Y} ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), AvailT = Avail band (bnot reg_bit(Reg)), + %% cp is two words: store y[Y] (offset << 2) at ?CP and y[Y+1] (Module*) + %% at ?CP_MODULE. I1 = ldr_y_reg(Reg, Y, AvailT), {BaseReg, Off} = ?CP, I2 = jit_xtensa_asm:s32i(Reg, BaseReg, Off), - Code = <>, + I3 = ldr_y_reg(Reg, Y + 1, AvailT), + {BaseRegM, OffM} = ?CP_MODULE, + I4 = jit_xtensa_asm:s32i(Reg, BaseRegM, OffM), + Code = <>, Stream1 = StreamModule:append(Stream0, Code), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), Regs2 = @@ -3077,10 +3032,10 @@ move_to_cp( State#state{stream = Stream1, regs = Regs2}. increment_sp( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail, regs = Regs0} = - State, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, Offset ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), {BaseReg1, Off1} = ?Y_REGS, I1 = jit_xtensa_asm:l32i(Reg, BaseReg1, Off1), @@ -3096,11 +3051,11 @@ set_continuation_to_label( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, regs = Regs0 } = State, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), %% In windowed ABI, the continuation will be called by C via CALL8, @@ -3123,12 +3078,12 @@ set_continuation_to_offset( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, branches = Branches, jump_table_start = JumpTableStart, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), OffsetRef = make_ref(), Offset = StreamModule:offset(Stream0), @@ -3158,11 +3113,10 @@ get_module_index( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, - used_regs = UsedRegs0, regs = Regs0 } = State ) -> + Avail = jit_regs:available_regs(Regs0), Reg = first_avail(Avail), RegBit = reg_bit(Reg), % Load module from jit_state (which is in a3) @@ -3174,13 +3128,76 @@ get_module_index( { State#state{ stream = Stream1, - available_regs = Avail band (bnot RegBit), - used_regs = UsedRegs0 bor RegBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. + +%% @doc Load the current module pointer (jit_state->module) into a fresh +%% register. Like get_module_index without the final module->index load, and +%% without caching the result. +-spec get_module(state()) -> {state(), xtensa_register()}. +get_module( + #state{ + stream_module = StreamModule, + stream = Stream0, + regs = Regs0 + } = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + % Load module from jit_state (which is in a3) + I1 = jit_xtensa_asm:l32i(Reg, ?JITSTATE_REG, ?JITSTATE_MODULE_OFFSET), + Stream1 = StreamModule:append(Stream0, I1), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. + +%% @doc Load the Module pointer stored in ctx->cp (?CP_MODULE) into a register. +-spec get_cp_module(state()) -> {state(), xtensa_register()}. +get_cp_module( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP_MODULE, + I = jit_xtensa_asm:l32i(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) }, Reg }. +%% @doc Load the offset word (offset << 2) stored in ctx->cp (?CP) into a register. +-spec get_cp_offset(state()) -> {state(), xtensa_register()}. +get_cp_offset( + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State +) -> + Avail = jit_regs:available_regs(Regs0), + Reg = first_avail(Avail), + RegBit = reg_bit(Reg), + {BaseReg, Off} = ?CP, + I = jit_xtensa_asm:l32i(Reg, BaseReg, Off), + Stream1 = StreamModule:append(Stream0, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + { + State#state{ + stream = Stream1, + regs = jit_regs:alloc_reg(Regs1, RegBit) + }, + Reg + }. and_( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, {free, Reg}, @@ -3204,78 +3221,109 @@ and_( {State0#state{stream = Stream1, regs = Regs1}, Reg}; % Xtensa has no andi instruction - small values fall through to general case and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val ) when - Val >= -2048 andalso Val =< 2047 andalso Avail =/= 0 + Val >= -2048 andalso Val =< 2047 -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:and_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, - {free, Reg}, - Val -) when Val < 0 andalso Val >= -256 andalso Avail =/= 0 -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:and_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}, Reg}; -and_( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, - {free, Reg}, - Val -) when Avail =/= 0 -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:and_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), - {State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}, Reg}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:and_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + {State1#state{stream = Stream2, regs = Regs1}, Reg}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:and_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }, + Reg + } + end; and_( - #state{available_regs = 0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val ) when Val < 0 andalso Val >= -256 -> - %% No available registers; use ?A8_REG (a8) directly as scratch. - %% a8 is not in AVAILABLE_REGS so it is the dedicated implicit scratch. - %% Avoid clobbering a0 (encoded return address required by RETW). - State1 = mov_immediate(State0, ?A8_REG, Val), - #state{stream_module = StreamModule, stream = Stream1} = State1, - I = jit_xtensa_asm:and_(Reg, Reg, ?A8_REG), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(State0#state.regs, Reg), - {State1#state{stream = Stream2, regs = Regs1}, Reg}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:and_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + {State1#state{stream = Stream2, regs = Regs1}, Reg}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:and_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }, + Reg + } + end; and_( - #state{available_regs = 0, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, {free, Reg}, Val ) -> - %% No available registers; use ?A8_REG (a8) directly as scratch. - State1 = mov_immediate(State0, ?A8_REG, Val), - #state{stream_module = StreamModule, stream = Stream1} = State1, - I = jit_xtensa_asm:and_(Reg, Reg, ?A8_REG), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(Regs0, Reg), - {State1#state{stream = Stream2, regs = Regs1}, Reg}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:and_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(Regs0, Reg), + {State1#state{stream = Stream2, regs = Regs1}, Reg}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:and_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Reg), Temp), + { + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }, + Reg + } + end; and_( - #state{stream_module = StreamModule, available_regs = Avail, used_regs = UR, regs = Regs0} = - State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, ?TERM_PRIMARY_CLEAR_MASK ) -> + Avail = jit_regs:available_regs(Regs0), ResultReg = case Avail of 0 -> ?A8_REG; @@ -3290,9 +3338,7 @@ and_( { State0#state{ stream = Stream1, - available_regs = Avail band (bnot ResultBit), - used_regs = UR bor ResultBit, - regs = Regs1 + regs = jit_regs:alloc_reg(Regs1, ResultBit) }, ResultReg }. @@ -3308,45 +3354,63 @@ or_(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State State0#state{stream = Stream1, regs = Regs1}; % Xtensa has no ori instruction - small values use li+or_ like general case or_( - #state{stream_module = StreamModule, available_regs = Avail} = State0, Reg, Val + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when - Val >= -2048 andalso Val =< 2047 andalso Avail =/= 0 + Val >= -2048 andalso Val =< 2047 -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:or_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - %% Take the cache from State1 (mov_immediate already invalidated Temp), - %% then also invalidate Reg whose value just got rewritten. - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}; -or_( - #state{available_regs = 0} = State0, Reg, Val -) -> - %% No available registers; use ?A8_REG (a8) directly as scratch. - %% a8 is not in AVAILABLE_REGS so it is the dedicated implicit scratch. - %% Avoid clobbering a0 (encoded return address required by RETW). - State1 = mov_immediate(State0, ?A8_REG, Val), - #state{stream_module = StreamModule, stream = Stream1} = State1, - I = jit_xtensa_asm:or_(Reg, Reg, ?A8_REG), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{stream = Stream2, regs = Regs1}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:or_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{stream = Stream2, regs = Regs1}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:or_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + } + end; or_( - #state{stream_module = StreamModule, available_regs = Avail} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:or_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:or_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{stream = Stream2, regs = Regs1}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:or_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + } + end. xor_( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, Reg, SrcReg @@ -3359,44 +3423,63 @@ xor_( State0#state{stream = Stream1, regs = Regs1}; % Xtensa has no xori instruction - small values use li+xor_ like general case xor_( - #state{stream_module = StreamModule, available_regs = Avail} = State0, Reg, Val + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when - Val >= -2048 andalso Val =< 2047 andalso Avail =/= 0 + Val >= -2048 andalso Val =< 2047 -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:xor_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - %% mov_immediate already invalidated Temp in State1#state.regs; also - %% invalidate Reg whose value just got rewritten. - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}; -xor_( - #state{available_regs = 0} = State0, Reg, Val -) -> - %% No available registers; use ?A8_REG (a8) directly as scratch. - %% Avoid clobbering a0 (encoded return address required by RETW). - State1 = mov_immediate(State0, ?A8_REG, Val), - #state{stream_module = StreamModule, stream = Stream1} = State1, - I = jit_xtensa_asm:xor_(Reg, Reg, ?A8_REG), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{stream = Stream2, regs = Regs1}; + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:xor_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{stream = Stream2, regs = Regs1}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:xor_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + } + end; xor_( - #state{stream_module = StreamModule, available_regs = Avail} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) -> - Temp = first_avail(Avail), - AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), - Stream1 = State1#state.stream, - I = jit_xtensa_asm:xor_(Reg, Reg, Temp), - Stream2 = StreamModule:append(Stream1, I), - Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + Avail = jit_regs:available_regs(Regs0), + case Avail of + 0 -> + State1 = mov_immediate(State0, ?A8_REG, Val), + #state{stream = Stream1} = State1, + I = jit_xtensa_asm:xor_(Reg, Reg, ?A8_REG), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{stream = Stream2, regs = Regs1}; + _ -> + Temp = first_avail(Avail), + AT = Avail band (bnot reg_bit(Temp)), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), + Stream1 = State1#state.stream, + I = jit_xtensa_asm:xor_(Reg, Reg, Temp), + Stream2 = StreamModule:append(Stream1, I), + Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + } + end. add(State0, {free, Reg}, Val) -> {add(State0, Reg, Val), Reg}; @@ -3414,20 +3497,25 @@ add(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State0#state{stream = Stream1, regs = Regs1}; -add(#state{stream_module = StreamModule, available_regs = Avail} = State0, Reg, Val) -> +add(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; _ -> first_avail(Avail) end, AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_xtensa_asm:add(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - %% mov_immediate already invalidated Temp in State1#state.regs. Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }. %% mov_immediate/2: returns binary, for inline code generation. %% Uses movi, movi+addmi, or movi+slli+addi sequences (no L32R). @@ -3700,26 +3788,32 @@ sub(#state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State Stream1 = StreamModule:append(Stream0, I), Regs1 = jit_regs:invalidate_reg(Regs0, Reg), State#state{stream = Stream1, regs = Regs1}; -sub(#state{stream_module = StreamModule, available_regs = Avail} = State0, Reg, Val) -> +sub(#state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; _ -> first_avail(Avail) end, AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_xtensa_asm:sub(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), - %% mov_immediate already invalidated Temp in State1#state.regs. Regs1 = jit_regs:invalidate_reg(State1#state.regs, Reg), - State1#state{available_regs = Avail, stream = Stream2, regs = Regs1}. + State1#state{ + stream = Stream2, + regs = jit_regs:set_available_regs(Regs1, Avail) + }. mul(State, _Reg, 1) -> State; mul(State, Reg, 2) -> shift_left(State, Reg, 1); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> +mul(#state{regs = Regs0} = State, Reg, 3) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -3732,7 +3826,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 3) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 4) -> shift_left(State, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> +mul(#state{regs = Regs0} = State, Reg, 5) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -3746,7 +3841,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 5) -> mul(State0, Reg, 6) -> State1 = mul(State0, Reg, 3), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> +mul(#state{regs = Regs0} = State, Reg, 7) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -3759,7 +3855,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 7) -> State#state{stream = Stream1, regs = Regs1}; mul(State, Reg, 8) -> shift_left(State, Reg, 3); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> +mul(#state{regs = Regs0} = State, Reg, 9) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -3773,7 +3870,8 @@ mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 9) -> mul(State0, Reg, 10) -> State1 = mul(State0, Reg, 5), mul(State1, Reg, 2); -mul(#state{available_regs = Avail, regs = Regs0} = State, Reg, 15) -> +mul(#state{regs = Regs0} = State, Reg, 15) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; @@ -3791,25 +3889,28 @@ mul(State, Reg, 32) -> mul(State, Reg, 64) -> shift_left(State, Reg, 6); mul( - #state{stream_module = StreamModule, available_regs = Avail, regs = Regs0} = State0, + #state{stream_module = StreamModule, regs = Regs0} = State0, Reg, Val ) when is_integer(Val) -> + Avail = jit_regs:available_regs(Regs0), Temp = case Avail of 0 -> ?A8_REG; _ -> first_avail(Avail) end, AT = Avail band (bnot reg_bit(Temp)), - State1 = mov_immediate(State0#state{available_regs = AT}, Temp, Val), + State1 = mov_immediate( + State0#state{regs = jit_regs:set_available_regs(Regs0, AT)}, Temp, Val + ), Stream1 = State1#state.stream, I = jit_xtensa_asm:mull(Reg, Reg, Temp), Stream2 = StreamModule:append(Stream1, I), Regs1 = jit_regs:invalidate_reg(jit_regs:invalidate_reg(Regs0, Temp), Reg), + State1Avail = jit_regs:available_regs(State1#state.regs), State1#state{ stream = Stream2, - available_regs = State1#state.available_regs bor reg_bit(Temp), - regs = Regs1 + regs = jit_regs:set_available_regs(Regs1, State1Avail bor reg_bit(Temp)) }; mul( #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State, DestReg, SrcReg @@ -3824,11 +3925,11 @@ decrement_reductions_and_maybe_schedule_next( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, jump_table_start = JumpTableStart, regs = Regs0 } = State0 ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), I1 = jit_xtensa_asm:l32i(Temp, ?JITSTATE_REG, ?JITSTATE_REDUCTIONCOUNT_OFFSET), I2 = jit_xtensa_asm:addi(Temp, Temp, -1), @@ -3860,9 +3961,11 @@ decrement_reductions_and_maybe_schedule_next( Stream3b, BNEOffset, <> ), StreamN = Stream4, - State3 = merge_used_regs(State2#state{stream = StreamN}, State1#state.used_regs), %% schedule_next clobbers caller-saved regs; invalidate cache at continuation. - State3#state{regs = jit_regs:invalidate_all(State1#state.regs)}. + State2#state{ + stream = StreamN, + regs = jit_regs:invalidate_all(State1#state.regs) + }. -spec call_or_schedule_next(state(), non_neg_integer()) -> state(). call_or_schedule_next(State0, Label) -> @@ -3874,11 +3977,11 @@ call_only_or_schedule_next( #state{ stream_module = StreamModule, stream = Stream0, - available_regs = Avail, regs = Regs0 } = State0a, Label ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), Regs1 = jit_regs:invalidate_reg(Regs0, Temp), State0 = State0a#state{regs = Regs1}, @@ -3935,32 +4038,32 @@ call_primitive_with_cp(State0, Primitive, Args) -> rewrite_cp_offset(State2, RewriteOffset, TempReg). -spec set_cp(state()) -> {state(), non_neg_integer(), xtensa_register()}. -set_cp(#state{available_regs = Avail, used_regs = UsedRegs} = State0) -> - TempReg = first_avail(Avail), - TempBit = reg_bit(TempReg), - %% Reserve TempReg for the offset BEFORE get_module_index consumes available registers. - State1 = State0#state{ - available_regs = Avail band (bnot TempBit), used_regs = UsedRegs bor TempBit - }, - {State2, Reg} = get_module_index(State1), - #state{stream_module = StreamModule, stream = Stream0} = State2, - - Offset = StreamModule:offset(Stream0), - I1 = jit_xtensa_asm:slli(Reg, Reg, 24), - %% Reserve 15 bytes for offset load (li generates 3..21 bytes, patched by rewrite_cp_offset). - I2 = +set_cp(State0) -> + %% cp is two words: store the Module pointer (jit_state->module) at + %% ?CP_MODULE, and the return offset << 2 at ?CP (patched by + %% rewrite_cp_offset below). + {#state{stream_module = StreamModule, stream = Stream0} = State1, ModReg} = + get_module(State0), + {BaseRegM, OffM} = ?CP_MODULE, + IModStore = jit_xtensa_asm:s32i(ModReg, BaseRegM, OffM), + Stream1 = StreamModule:append(Stream0, IModStore), + State2 = free_native_register(State1#state{stream = Stream1}, ModReg), + %% Get a temporary register to hold the offset value + TempReg = first_avail(jit_regs:available_regs(State2#state.regs)), + Offset = StreamModule:offset(Stream1), + %% Reserve 15 bytes for offset load (li generates 3..21 bytes, patched by + %% rewrite_cp_offset). The placeholder is loaded into TempReg, which is then + %% stored to ?CP. + I1 = <<16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF, 16#FF>>, - MOVOffset = Offset + byte_size(I1), - I4 = jit_xtensa_asm:or_(Reg, Reg, TempReg), + MOVOffset = Offset, {BaseReg, Off} = ?CP, - I5 = jit_xtensa_asm:s32i(Reg, BaseReg, Off), - Code = <>, - Stream1 = StreamModule:append(Stream0, Code), - State3 = State2#state{stream = Stream1}, - State4 = free_native_register(State3, Reg), - State5 = free_native_register(State4, TempReg), - {State5, MOVOffset, TempReg}. + I2 = jit_xtensa_asm:s32i(TempReg, BaseReg, Off), + Code = <>, + Stream2 = StreamModule:append(Stream1, Code), + State3 = State2#state{stream = Stream2}, + {State3, MOVOffset, TempReg}. -spec rewrite_cp_offset(state(), non_neg_integer(), xtensa_register()) -> state(). rewrite_cp_offset( @@ -3988,9 +4091,10 @@ rewrite_cp_offset( State0#state{stream = Stream2}. set_bs( - #state{stream_module = StreamModule, stream = Stream0, available_regs = Avail} = State0, + #state{stream_module = StreamModule, stream = Stream0, regs = Regs0} = State0, TermReg ) -> + Avail = jit_regs:available_regs(Regs0), Temp = first_avail(Avail), {BaseReg1, Off1} = ?BS, I1 = jit_xtensa_asm:s32i(TermReg, BaseReg1, Off1), @@ -4203,80 +4307,6 @@ reg_bit(a13) -> ?REG_BIT_A13; reg_bit(a14) -> ?REG_BIT_A14; reg_bit(a15) -> ?REG_BIT_A15. -%% High registers (a15-a9) are clobbered by CALLX8 so prefer them to minimize saves. -first_avail(Mask) when Mask band ?REG_BIT_A15 =/= 0 -> a15; -first_avail(Mask) when Mask band ?REG_BIT_A14 =/= 0 -> a14; -first_avail(Mask) when Mask band ?REG_BIT_A13 =/= 0 -> a13; -first_avail(Mask) when Mask band ?REG_BIT_A12 =/= 0 -> a12; -first_avail(Mask) when Mask band ?REG_BIT_A11 =/= 0 -> a11; -first_avail(Mask) when Mask band ?REG_BIT_A10 =/= 0 -> a10; -first_avail(Mask) when Mask band ?REG_BIT_A9 =/= 0 -> a9; -first_avail(Mask) when Mask band ?REG_BIT_A8 =/= 0 -> a8; -first_avail(Mask) when Mask band ?REG_BIT_A7 =/= 0 -> a7; -first_avail(Mask) when Mask band ?REG_BIT_A6 =/= 0 -> a6; -first_avail(Mask) when Mask band ?REG_BIT_A5 =/= 0 -> a5. - -mask_to_list(0) -> []; -mask_to_list(Mask) -> mask_to_list_a15(Mask). - -mask_to_list_a15(Mask) when Mask band ?REG_BIT_A15 =/= 0 -> [a15 | mask_to_list_a14(Mask)]; -mask_to_list_a15(Mask) -> mask_to_list_a14(Mask). -mask_to_list_a14(Mask) when Mask band ?REG_BIT_A14 =/= 0 -> [a14 | mask_to_list_a13(Mask)]; -mask_to_list_a14(Mask) -> mask_to_list_a13(Mask). -mask_to_list_a13(Mask) when Mask band ?REG_BIT_A13 =/= 0 -> [a13 | mask_to_list_a12(Mask)]; -mask_to_list_a13(Mask) -> mask_to_list_a12(Mask). -mask_to_list_a12(Mask) when Mask band ?REG_BIT_A12 =/= 0 -> [a12 | mask_to_list_a11(Mask)]; -mask_to_list_a12(Mask) -> mask_to_list_a11(Mask). -mask_to_list_a11(Mask) when Mask band ?REG_BIT_A11 =/= 0 -> [a11 | mask_to_list_a10(Mask)]; -mask_to_list_a11(Mask) -> mask_to_list_a10(Mask). -mask_to_list_a10(Mask) when Mask band ?REG_BIT_A10 =/= 0 -> [a10 | mask_to_list_a9(Mask)]; -mask_to_list_a10(Mask) -> mask_to_list_a9(Mask). -mask_to_list_a9(Mask) when Mask band ?REG_BIT_A9 =/= 0 -> [a9 | mask_to_list_a8(Mask)]; -mask_to_list_a9(Mask) -> mask_to_list_a8(Mask). -mask_to_list_a8(Mask) when Mask band ?REG_BIT_A8 =/= 0 -> [a8 | mask_to_list_a7(Mask)]; -mask_to_list_a8(Mask) -> mask_to_list_a7(Mask). -mask_to_list_a7(Mask) when Mask band ?REG_BIT_A7 =/= 0 -> [a7 | mask_to_list_a6(Mask)]; -mask_to_list_a7(Mask) -> mask_to_list_a6(Mask). -mask_to_list_a6(Mask) when Mask band ?REG_BIT_A6 =/= 0 -> [a6 | mask_to_list_a5(Mask)]; -mask_to_list_a6(Mask) -> mask_to_list_a5(Mask). -mask_to_list_a5(Mask) when Mask band ?REG_BIT_A5 =/= 0 -> [a5 | mask_to_list_a4(Mask)]; -mask_to_list_a5(Mask) -> mask_to_list_a4(Mask). -mask_to_list_a4(Mask) when Mask band ?REG_BIT_A4 =/= 0 -> [a4 | mask_to_list_a3(Mask)]; -mask_to_list_a4(Mask) -> mask_to_list_a3(Mask). -mask_to_list_a3(Mask) when Mask band ?REG_BIT_A3 =/= 0 -> [a3 | mask_to_list_a2(Mask)]; -mask_to_list_a3(Mask) -> mask_to_list_a2(Mask). -mask_to_list_a2(Mask) when Mask band ?REG_BIT_A2 =/= 0 -> [a2 | mask_to_list_a1(Mask)]; -mask_to_list_a2(Mask) -> mask_to_list_a1(Mask). -mask_to_list_a1(Mask) when Mask band ?REG_BIT_A1 =/= 0 -> [a1 | mask_to_list_a0(Mask)]; -mask_to_list_a1(Mask) -> mask_to_list_a0(Mask). -mask_to_list_a0(Mask) when Mask band ?REG_BIT_A0 =/= 0 -> [a0]; -mask_to_list_a0(_Mask) -> []. - -args_regs(Args) -> - lists:map( - fun - ({free, {ptr, Reg}}) -> Reg; - ({free, Reg}) when is_atom(Reg) -> Reg; - ({free, Imm}) when is_integer(Imm) -> imm; - (offset) -> imm; - (ctx) -> ?CTX_REG; - (jit_state) -> jit_state; - (jit_state_tail_call) -> jit_state; - (stack) -> stack; - (Reg) when is_atom(Reg) -> Reg; - (Imm) when is_integer(Imm) -> imm; - ({ptr, Reg}) -> Reg; - ({x_reg, _}) -> ?CTX_REG; - ({y_reg, _}) -> ?CTX_REG; - ({fp_reg, _}) -> ?CTX_REG; - ({free, {x_reg, _}}) -> ?CTX_REG; - ({free, {y_reg, _}}) -> ?CTX_REG; - ({free, {fp_reg, _}}) -> ?CTX_REG; - ({avm_int64_t, _}) -> imm - end, - Args - ). - %%----------------------------------------------------------------------------- %% @doc Add a label at the current offset. %% @end diff --git a/src/libAtomVM/context.c b/src/libAtomVM/context.c index b8b29273d5..b543bad8b2 100644 --- a/src/libAtomVM/context.c +++ b/src/libAtomVM/context.c @@ -488,8 +488,8 @@ void context_process_code_server_resume_signal(Context *ctx) #endif #endif // Fix CP to OP_INT_CALL_END - if (ctx->cp == module_address(module->module_index, 0)) { - ctx->cp = module_address(module->module_index, module->end_instruction_ii); + if (ctx->cp == make_cp(module, 0)) { + ctx->cp = make_cp(module, module->end_instruction_ii); } #endif context_update_flags(ctx, ~Trap, NoFlags); @@ -1206,7 +1206,10 @@ COLD_FUNC void context_dump(Context *ctx) Module *cp_mod; int label; size_t offset; - module_cp_to_label_offset(*ct, &cp_mod, &label, &offset, NULL, ctx->global); + // A saved cp spans CP_SIZE_IN_TERMS slots (2 on 32-bit: offset then + // Module*). Reconstruct it from all its slots and skip the extra one. + module_cp_to_label_offset(load_cp(ct), &cp_mod, &label, &offset, NULL, ctx->global); + ct += CP_SIZE_IN_TERMS - 1; // Cast offset to unsigned as some embedded libc implementations do not support %zu fprintf(stderr, "#CP\n", cp_mod->module_index, label, (unsigned) offset); diff --git a/src/libAtomVM/context.h b/src/libAtomVM/context.h index e43f360620..aacff19e91 100644 --- a/src/libAtomVM/context.h +++ b/src/libAtomVM/context.h @@ -95,7 +95,17 @@ struct Context // Following fields offsets are also hard-coded in jit backends term x[MAX_REG + 1]; - term cp; + // Continuation pointer: 64-bit wide, so on 32-bit term platforms it spans + // two stack slots and can hold a full Module pointer plus a code offset. + // On 32-bit term platforms uint64_t is naturally 8-byte aligned, which would + // insert 4 bytes of padding after x[] and push cp to 0x60; the JIT backends + // expect the offset word at 0x5C and the Module* at 0x60, so keep it packed + // and 4-byte aligned (it always lands on a 4-byte boundary anyway). +#if TERM_BYTES == 4 + cp_t cp __attribute__((packed, aligned(4))); +#else + cp_t cp; +#endif avm_float_t *fr; term bs; size_t bs_offset; diff --git a/src/libAtomVM/jit.c b/src/libAtomVM/jit.c index 8d8bd5b4f8..9f62681553 100644 --- a/src/libAtomVM/jit.c +++ b/src/libAtomVM/jit.c @@ -161,10 +161,12 @@ _Static_assert(offsetof(JITState, remaining_reductions) == 0x10, "jit_state->rem #elif JIT_ARCH_TARGET == JIT_ARCH_ARMV6M || JIT_ARCH_TARGET == JIT_ARCH_ARM32 || JIT_ARCH_TARGET == JIT_ARCH_RISCV32 || JIT_ARCH_TARGET == JIT_ARCH_WASM32 || JIT_ARCH_TARGET == JIT_ARCH_XTENSA _Static_assert(offsetof(Context, e) == 0x14, "ctx->e is 0x14 in 32-bit backends"); _Static_assert(offsetof(Context, x) == 0x18, "ctx->x is 0x18 in 32-bit backends"); +// cp is now a 64-bit cp_t spanning two 32-bit words (low word = offset << 2 at +// 0x5C, high word = Module* at 0x60); the following fields shift up by one word. _Static_assert(offsetof(Context, cp) == 0x5C, "ctx->cp is 0x5C in 32-bit backends"); -_Static_assert(offsetof(Context, fr) == 0x60, "ctx->fr is 0x60 in 32-bit backends"); -_Static_assert(offsetof(Context, bs) == 0x64, "ctx->bs is 0x64 in 32-bit backends"); -_Static_assert(offsetof(Context, bs_offset) == 0x68, "ctx->bs_offset is 0x68 in 32-bit backends"); +_Static_assert(offsetof(Context, fr) == 0x64, "ctx->fr is 0x64 in 32-bit backends"); +_Static_assert(offsetof(Context, bs) == 0x68, "ctx->bs is 0x68 in 32-bit backends"); +_Static_assert(offsetof(Context, bs_offset) == 0x6C, "ctx->bs_offset is 0x6C in 32-bit backends"); _Static_assert(offsetof(JITState, module) == 0x0, "jit_state->module is 0x0 in 32-bit backends"); _Static_assert(offsetof(JITState, continuation) == 0x4, "jit_state->continuation is 0x4 in 32-bit backends"); @@ -260,26 +262,25 @@ static void jit_trim_live_regs(Context *ctx, uint32_t live) // Update jit_state->module and jit_state->continuation static Context *jit_return(Context *ctx, JITState *jit_state) { - int module_index = ctx->cp >> 24; - TRACE("jit_return: ctx->cp = %d, module_index = %d, offset = %d\n", (int) ctx->cp, module_index, (int) (ctx->cp & 0xFFFFFF) >> 2); - Module *mod = globalcontext_get_module_by_index(ctx->global, module_index); + Module *mod = cp_to_module(ctx->cp, ctx->global); + unsigned int offset = cp_to_offset(ctx->cp); + TRACE("jit_return: mod = %p, offset = %u\n", (void *) mod, offset); // Native case #ifndef AVM_NO_EMU if (mod->native_code == NULL) { // return to emulated const uint8_t *code = mod->code->code; - const uint8_t *pc = code + ((ctx->cp & 0xFFFFFF) >> 2); - jit_state->continuation_pc = pc; + jit_state->continuation_pc = code + offset; } else { #endif #ifdef JIT_JUMPTABLE_IS_DATA // WASM: continuation stores (label + 1) for the dispatch loop to convert. - int label = ((ctx->cp & 0xFFFFFF) >> 2) / JIT_JUMPTABLE_ENTRY_SIZE; - TRACE("jit_return: cp=0x%x mod=%d label=%d\n", (unsigned) ctx->cp, module_index, label); + int label = (int) offset / JIT_JUMPTABLE_ENTRY_SIZE; + TRACE("jit_return: mod=%p label=%d\n", (void *) mod, label); jit_state->continuation = (NativeContinuation) (label + 1); #else - uintptr_t native_pc = (uintptr_t) mod->native_code + ((ctx->cp & 0xFFFFFF) >> 2); + uintptr_t native_pc = (uintptr_t) mod->native_code + offset; jit_state->continuation = (NativeContinuation) native_pc; #endif #ifndef AVM_NO_EMU @@ -510,8 +511,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int // workaround for issue // https://github.com/erlang/otp/issues/7152 if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } if (ctx->heap.root->next) { @@ -519,7 +520,7 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); } } - if ((long) ctx->cp == -1) { + if (cp_is_terminate(ctx->cp)) { return 0; } @@ -531,8 +532,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int // not access ctx->e or ctx->cp) if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } // Native case @@ -567,8 +568,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int } case ModuleNativeFunction: { if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); @@ -582,8 +583,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int } case BIFFunctionType: { if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func); @@ -615,8 +616,8 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int // even on OTP28, so it is required to allow calling them using // CALL_EXT_ONLY even on OTP28: BIFs are used for try ... catch. if (n_words >= 0) { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); } const struct GCBif *gcbif = EXPORTED_FUNCTION_TO_GCBIF(func); @@ -653,15 +654,15 @@ static term jit_module_get_atom_term_by_id(JITState *jit_state, int atom_index) static bool jit_allocate(Context *ctx, JITState *jit_state, uint32_t stack_need, uint32_t heap_need, uint32_t live) { TRACE("jit_allocate: ENTRY ctx=%p jit_state=%p stack_need=%" PRIu32 " heap_need=%" PRIu32 " live=%" PRIu32 "\n", (void *) ctx, (void *) jit_state, stack_need, heap_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need > ctx->e - (stack_need + 1)))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need > ctx->e - (stack_need + CP_SIZE_IN_TERMS)))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + 1, live, ctx->x, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + CP_SIZE_IN_TERMS, live, ctx->x, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { set_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); return false; } } - ctx->e -= stack_need + 1; - ctx->e[stack_need] = ctx->cp; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; + store_cp(ctx->e + stack_need, ctx->cp); return true; } @@ -676,8 +677,8 @@ static BifImpl0 jit_get_imported_bif(JITState *jit_state, uint32_t bif) static bool jit_deallocate(Context *ctx, JITState *jit_state, uint32_t n_words) { TRACE("jit_deallocate: n_words=%" PRIu32 "\n", n_words); - ctx->cp = ctx->e[n_words]; - ctx->e += n_words + 1; + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += n_words + CP_SIZE_IN_TERMS; // Hopefully, we only need x[0] if (ctx->heap.root->next) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { diff --git a/src/libAtomVM/module.c b/src/libAtomVM/module.c index eecb92f940..c5364459bf 100644 --- a/src/libAtomVM/module.c +++ b/src/libAtomVM/module.c @@ -1117,6 +1117,10 @@ Module *module_new_from_iff_binary(GlobalContext *global, const void *iff_binary fprintf(stderr, "Error: Failed to allocate memory: %s:%i.\n", __FILE__, __LINE__); return NULL; } + // On 32-bit, continuation pointers store this Module pointer directly in a + // stack slot, relying on its low 2 bits being clear (TERM_PRIMARY_CP tag) so + // the GC skips it. malloc guarantees suitable alignment; assert it anyway. + assert(((uintptr_t) mod & TERM_PRIMARY_MASK) == 0); memset(mod, 0, sizeof(Module)); mod->module_index = -1; @@ -2104,10 +2108,10 @@ bool module_find_line(Module *mod, size_t offset, uint32_t *line, size_t *filena #endif } -COLD_FUNC void module_cp_to_label_offset(term cp, Module **cp_mod, int *label, size_t *l_off, size_t *out_mod_offset, GlobalContext *global) +COLD_FUNC void module_cp_to_label_offset(cp_t cp, Module **cp_mod, int *label, size_t *l_off, size_t *out_mod_offset, GlobalContext *global) { - Module *mod = globalcontext_get_module_by_index(global, ((uintptr_t) cp) >> 24); - size_t mod_offset = (cp & 0xFFFFFF) >> 2; + Module *mod = cp_to_module(cp, global); + size_t mod_offset = cp_to_offset(cp); if (out_mod_offset) { *out_mod_offset = mod_offset; } diff --git a/src/libAtomVM/module.h b/src/libAtomVM/module.h index 02f4d84fc6..e87dad3b41 100644 --- a/src/libAtomVM/module.h +++ b/src/libAtomVM/module.h @@ -330,16 +330,118 @@ static inline const struct ExportedFunction *module_resolve_function(Module *mod } /* - * @brief Casts an instruction index and module index to a return address + * Number of `term` stack slots occupied by a saved continuation pointer. + * On 64-bit a cp fits in one slot; on 32-bit it spans two (offset + Module*). + */ +#if TERM_BITS == 64 +#define CP_SIZE_IN_TERMS 1 +#else +#define CP_SIZE_IN_TERMS 2 +#endif + +/* + * @brief Builds a continuation pointer (return address) from a module and an instruction index. + * + * @details The continuation pointer encodes which module to resume and at which + * code offset. The offset is shifted left by 2 (TERM_PRIMARY_CP tag, 0b00) so the + * garbage collector skips it when it appears on the stack. The offset is mode- + * interpreted at the destination module (BEAM bytecode offset when emulated, + * native code offset when jit-compiled). + * + * On 64-bit, the module is identified by its index packed in the high bits (as + * historically). On 32-bit, the Module pointer is stored directly in the high 32 + * bits: AtomVM never unloads modules so the pointer is stable, this removes the + * 256-module ceiling of index packing, and the return path needs no index lookup. + * A malloc-aligned Module pointer has its low 2 bits clear, so once stored in its + * own stack slot it carries the TERM_PRIMARY_CP tag and is also skipped by the GC. * - * @details Casts an instruction index and module index to a value that return instruction can restore later. - * @param module_index the module index - * @param the instruction index (0 is the first module instruction) - * @return casted return address + * @param mod the module to resume into + * @param instruction_index the code offset (0 is the first module instruction) + * @return the continuation pointer + */ +static inline cp_t make_cp(const Module *mod, unsigned int instruction_index) +{ +#if TERM_BITS == 64 + return ((cp_t) (unsigned int) mod->module_index << 24) | ((cp_t) instruction_index << 2); +#else + return ((cp_t) (uintptr_t) mod << 32) | ((cp_t) (instruction_index << 2)); +#endif +} + +/* + * @brief Builds a continuation pointer from a module index and instruction index. + * + * @details Cold-path counterpart of make_cp() for callers that only have a module + * index (e.g. rebuilding a cp from a stored raw stacktrace). On 32-bit it resolves + * the index to a Module pointer; on 64-bit it packs the index as make_cp() would. + */ +static inline cp_t make_cp_from_index(unsigned int module_index, unsigned int instruction_index, GlobalContext *global) +{ +#if TERM_BITS == 64 + (void) global; + return ((cp_t) module_index << 24) | ((cp_t) instruction_index << 2); +#else + Module *mod = globalcontext_get_module_by_index(global, (int) module_index); + return make_cp(mod, instruction_index); +#endif +} + +/* + * @brief Stores a continuation pointer to a stack location (CP_SIZE_IN_TERMS slots). + */ +static inline void store_cp(term *dst, cp_t cp) +{ +#if TERM_BITS == 64 + dst[0] = (term) cp; +#else + dst[0] = (term) (uint32_t) cp; // offset << 2 (TERM_PRIMARY_CP tag) + dst[1] = (term) (uintptr_t) (cp >> 32); // Module pointer (aligned, TERM_PRIMARY_CP tag) +#endif +} + +/* + * @brief Loads a continuation pointer previously saved with store_cp. + */ +static inline cp_t load_cp(const term *src) +{ +#if TERM_BITS == 64 + return (cp_t) src[0]; +#else + return (cp_t) (uint32_t) src[0] | ((cp_t) (uintptr_t) src[1] << 32); +#endif +} + +/* + * @brief Returns the module a continuation pointer resumes into. + */ +static inline Module *cp_to_module(cp_t cp, GlobalContext *global) +{ +#if TERM_BITS == 64 + return globalcontext_get_module_by_index(global, (int) (cp >> 24)); +#else + (void) global; + return (Module *) (uintptr_t) (cp >> 32); +#endif +} + +/* + * @brief Returns the code offset a continuation pointer resumes at. + */ +static inline unsigned int cp_to_offset(cp_t cp) +{ +#if TERM_BITS == 64 + return (unsigned int) ((cp & 0xFFFFFF) >> 2); +#else + return (unsigned int) ((uint32_t) cp >> 2); +#endif +} + +/* + * @brief Tells whether a continuation pointer is the process-termination sentinel. */ -static inline term module_address(unsigned int module_index, unsigned int instruction_index) +static inline bool cp_is_terminate(cp_t cp) { - return (term) ((module_index << 24) | (instruction_index << 2)); + return ((int64_t) cp) == -1; } static inline uint32_t module_get_fun_freeze(const Module *this_module, int fun_index) @@ -477,7 +579,7 @@ static inline bool module_has_line_chunk(Module *mod) * @param mod_offset if not null, set to offset of cp from module start * @param global the global context */ -void module_cp_to_label_offset(term cp, Module **cp_mod, int *label, size_t *l_off, size_t *mod_offset, GlobalContext *global); +void module_cp_to_label_offset(cp_t cp, Module **cp_mod, int *label, size_t *l_off, size_t *mod_offset, GlobalContext *global); /** * @brief Get the offset of a given label from the beginning of the code, emulated or native diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c index 2e3f70f7eb..4f46c83914 100644 --- a/src/libAtomVM/nifs.c +++ b/src/libAtomVM/nifs.c @@ -1637,7 +1637,7 @@ static term nif_erlang_spawn_fun_opt(Context *ctx, int argc, term argv[]) #ifndef AVM_NO_JIT } #endif - new_ctx->cp = module_address(fun_module->module_index, fun_module->end_instruction_ii); + new_ctx->cp = make_cp(fun_module, fun_module->end_instruction_ii); return do_spawn(ctx, new_ctx, arity, n_freeze, opts_term); } @@ -1694,7 +1694,7 @@ term nif_erlang_spawn_opt(Context *ctx, int argc, term argv[]) #ifndef AVM_NO_JIT } #endif - new_ctx->cp = module_address(found_module->module_index, found_module->end_instruction_ii); + new_ctx->cp = make_cp(found_module, found_module->end_instruction_ii); // TODO: check available registers count int reg_index = 0; diff --git a/src/libAtomVM/opcodesswitch.h b/src/libAtomVM/opcodesswitch.h index 889ff319a8..3de039e9f2 100644 --- a/src/libAtomVM/opcodesswitch.h +++ b/src/libAtomVM/opcodesswitch.h @@ -979,9 +979,12 @@ static void destroy_extended_registers(Context *ctx, unsigned int live) } \ } -#if AVM_NO_JIT - -#define DO_RETURN() \ +// Resolve the destination module of a return from ctx->cp, keeping the +// prev_mod/mod cache so an intra-module return needs no lookup. On 64-bit the +// module is identified by its packed index (historical layout); on 32-bit the +// cp carries the Module pointer directly (high word), so no index lookup at all. +#if TERM_BITS == 64 +#define DO_RETURN_RESOLVE_MODULE() \ { \ int module_index = ((uintptr_t) ctx->cp) >> 24; \ if (module_index == prev_mod->module_index) { \ @@ -994,7 +997,30 @@ static void destroy_extended_registers(Context *ctx, unsigned int live) mod = globalcontext_get_module_by_index(glb, module_index); \ code = mod->code->code; \ } \ - pc = code + ((((uintptr_t) ctx->cp) & 0xFFFFFF) >> 2); \ + } +#else +#define DO_RETURN_RESOLVE_MODULE() \ + { \ + Module *cp_mod = (Module *) (uintptr_t) (ctx->cp >> 32); \ + if (cp_mod == prev_mod) { \ + Module *t = mod; \ + mod = prev_mod; \ + prev_mod = t; \ + code = mod->code->code; \ + } else if (cp_mod != mod) { \ + prev_mod = mod; \ + mod = cp_mod; \ + code = mod->code->code; \ + } \ + } +#endif + +#if AVM_NO_JIT + +#define DO_RETURN() \ + { \ + DO_RETURN_RESOLVE_MODULE(); \ + pc = code + cp_to_offset(ctx->cp); \ } #else @@ -1002,35 +1028,25 @@ static void destroy_extended_registers(Context *ctx, unsigned int live) #ifdef JIT_JUMPTABLE_IS_DATA static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) { - int label = (int) ((ctx->cp & 0xFFFFFF) >> 2) / JIT_JUMPTABLE_ENTRY_SIZE; + int label = (int) cp_to_offset(ctx->cp) / JIT_JUMPTABLE_ENTRY_SIZE; return module_get_native_entry_point(mod, label); } #else static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) { return (ModuleNativeEntryPoint) ((const uint8_t *) mod->native_code) - + ((ctx->cp & 0xFFFFFF) >> 2); + + cp_to_offset(ctx->cp); } #endif #define DO_RETURN() \ { \ - int module_index = ((uintptr_t) ctx->cp) >> 24; \ - if (module_index == prev_mod->module_index) { \ - Module *t = mod; \ - mod = prev_mod; \ - prev_mod = t; \ - code = mod->code->code; \ - } else if (module_index != mod->module_index) { \ - prev_mod = mod; \ - mod = globalcontext_get_module_by_index(glb, module_index); \ - code = mod->code->code; \ - } \ + DO_RETURN_RESOLVE_MODULE(); \ if (mod->native_code) { \ native_pc = do_return_native(mod, ctx); \ } else { \ native_pc = NULL; \ - pc = code + ((((uintptr_t) ctx->cp) & 0xFFFFFF) >> 2); \ + pc = code + cp_to_offset(ctx->cp); \ } \ } @@ -1148,7 +1164,7 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) dreg_t ext_reg = extended_register_ptr(ctx, i); \ *ext_reg = boxed_value[i - fun_arity + 3]; \ } \ - ctx->cp = module_address(mod->module_index, pc - code); \ + ctx->cp = make_cp(mod, pc - code); \ JUMP_TO_LABEL(fun_module, label); #define DECODE_FLAGS_LIST(flags_value, flags, opcode) \ @@ -1620,7 +1636,7 @@ int context_execute_loop(Context *ctx, Module *mod, const char *function_name, i return 0; } - ctx->cp = module_address(mod->module_index, mod->end_instruction_ii); + ctx->cp = make_cp(mod, mod->end_instruction_ii); ctx->saved_module = mod; #if AVM_NO_JIT @@ -1832,7 +1848,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("call/2, arity=%i, label=%i\n", arity, label); USED_BY_TRACE(arity); - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); remaining_reductions--; if (LIKELY(remaining_reductions)) { @@ -1856,8 +1872,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("call_last/3, arity=%i, label=%i, dellocate=%i\n", arity, label, n_words); USED_BY_TRACE(arity); - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); DEBUG_DUMP_STACK(ctx); @@ -1929,7 +1945,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); JUMP_TO_LABEL(jump->target, jump->label); break; @@ -1938,7 +1954,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleNativeFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); if (jump->target != mod) { prev_mod = mod; mod = jump->target; @@ -2041,8 +2057,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // workaround for issue // https://github.com/erlang/otp/issues/7152 - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); if (ctx->heap.root->next) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { @@ -2058,8 +2074,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // (and it doesn't matter as the code below does // not access ctx->e or ctx->cp) - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); JUMP_TO_LABEL(jump->target, jump->label); @@ -2070,8 +2086,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleNativeFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); if (jump->target != mod) { prev_mod = mod; @@ -2083,8 +2099,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif case BIFFunctionType: { - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func); term return_value; @@ -2116,8 +2132,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // Regular CALL_EXT_LASTs to those functions are generated as well // even on OTP28, so it is required to allow calling them using // CALL_EXT_LAST even on OTP28: BIFs are used for try ... catch. - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); const struct GCBif *gcbif = EXPORTED_FUNCTION_TO_GCBIF(func); term return_value; @@ -2231,14 +2247,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(live, pc); TRACE("allocate/2 stack_need=%i, live=%i\n", stack_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + 1)))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + CP_SIZE_IN_TERMS)))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + 1, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + CP_SIZE_IN_TERMS, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - ctx->e -= stack_need + 1; - ctx->e[stack_need] = ctx->cp; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; + store_cp(ctx->e + stack_need, ctx->cp); break; } @@ -2251,14 +2267,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(live, pc); TRACE("allocate_heap/2 stack_need=%i, heap_need=%i, live=%i\n", stack_need, heap_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need) > ctx->e - (stack_need + 1))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr + heap_need) > ctx->e - (stack_need + CP_SIZE_IN_TERMS))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + 1, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, heap_need + stack_need + CP_SIZE_IN_TERMS, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - ctx->e -= stack_need + 1; - ctx->e[stack_need] = ctx->cp; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; + store_cp(ctx->e + stack_need, ctx->cp); break; } @@ -2269,18 +2285,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LITERAL(live, pc); TRACE("allocate_zero/2 stack_need=%i, live=%i\n", stack_need, live); - if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + 1)))) { + if (ctx->heap.root->next || ((ctx->heap.heap_ptr > ctx->e - (stack_need + CP_SIZE_IN_TERMS)))) { TRIM_LIVE_REGS(live); - if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + 1, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + if (UNLIKELY(memory_ensure_free_with_roots(ctx, stack_need + CP_SIZE_IN_TERMS, live, x_regs, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - ctx->e -= stack_need + 1; + ctx->e -= stack_need + CP_SIZE_IN_TERMS; for (uint32_t s = 0; s < stack_need; s++) { ctx->e[s] = term_nil(); } - ctx->e[stack_need] = ctx->cp; + store_cp(ctx->e + stack_need, ctx->cp); break; } @@ -2329,8 +2345,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DEBUG_DUMP_STACK(ctx); - ctx->cp = ctx->e[n_words]; - ctx->e += n_words + 1; + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += n_words + CP_SIZE_IN_TERMS; DEBUG_DUMP_STACK(ctx); // Hopefully, we only need x[0] if (ctx->heap.root->next) { @@ -2346,7 +2362,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE_RETURN(ctx); - if ((intptr_t) ctx->cp == -1) { + if (cp_is_terminate(ctx->cp)) { return 0; } @@ -3140,7 +3156,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } } - if ((intptr_t) ctx->cp == -1) { + if (cp_is_terminate(ctx->cp)) { return 0; } @@ -4558,7 +4574,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) SET_ERROR(UNDEF_ATOM); HANDLE_ERROR(); } - ctx->cp = module_address(mod->module_index, pc - code); + ctx->cp = make_cp(mod, pc - code); JUMP_TO_LABEL(target_module, target_label); } break; @@ -4583,8 +4599,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(BADARG_ATOM); } - ctx->cp = ctx->e[n_words]; - ctx->e += (n_words + 1); + ctx->cp = load_cp(ctx->e + n_words); + ctx->e += (n_words + CP_SIZE_IN_TERMS); atom_index_t module_name = term_to_atom_index(module); atom_index_t function_name = term_to_atom_index(function); diff --git a/src/libAtomVM/stacktrace.c b/src/libAtomVM/stacktrace.c index 42fe665bc3..a47cda9bea 100644 --- a/src/libAtomVM/stacktrace.c +++ b/src/libAtomVM/stacktrace.c @@ -163,7 +163,10 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, Module *cp_mod; size_t mod_offset; - module_cp_to_label_offset(*ct, &cp_mod, NULL, NULL, &mod_offset, ctx->global); + // A saved cp spans CP_SIZE_IN_TERMS slots (2 on 32-bit: offset then + // Module*). Both are CP-tagged; reconstruct it and skip the extra slot. + module_cp_to_label_offset(load_cp(ct), &cp_mod, NULL, NULL, &mod_offset, ctx->global); + ct += CP_SIZE_IN_TERMS - 1; // TODO: investigate // mod_offset is currently never equal to cp_mod->end_instruction_ii with native code if (mod_offset != cp_mod->end_instruction_ii && !(prev_mod == cp_mod && mod_offset == prev_mod_offset)) { @@ -298,7 +301,8 @@ term stacktrace_create_raw_mfa(Context *ctx, Module *mod, size_t current_offset, Module *cp_mod; size_t mod_offset; - module_cp_to_label_offset(*ct, &cp_mod, NULL, NULL, &mod_offset, ctx->global); + module_cp_to_label_offset(load_cp(ct), &cp_mod, NULL, NULL, &mod_offset, ctx->global); + ct += CP_SIZE_IN_TERMS - 1; if (mod_offset != cp_mod->end_instruction_ii && !(prev_mod == cp_mod && mod_offset == prev_mod_offset)) { prev_mod = cp_mod; @@ -428,9 +432,10 @@ term stacktrace_build(Context *ctx, term *stack_info, uint32_t live) size_t mod_index_tuple_arity = term_get_tuple_arity(mod_index_tuple); assert((mod_index_tuple_arity == 2) || (mod_index_tuple_arity == 5)); - term cp = module_address( + cp_t cp = make_cp_from_index( term_to_int(term_get_tuple_element(mod_index_tuple, 0)), - term_to_int(term_get_tuple_element(mod_index_tuple, 1))); + term_to_int(term_get_tuple_element(mod_index_tuple, 1)), + ctx->global); Module *cp_mod; int label; diff --git a/src/libAtomVM/term_typedef.h b/src/libAtomVM/term_typedef.h index be4ce9bcca..b0f97119d9 100644 --- a/src/libAtomVM/term_typedef.h +++ b/src/libAtomVM/term_typedef.h @@ -45,6 +45,17 @@ typedef uintptr_t term; #define TERM_U_FMT PRIuPTR #define TERM_X_FMT PRIXPTR +/** + * A continuation pointer (return address). It packs a module identifier and an + * instruction/code offset. It is always 64 bits wide so that on 32-bit `term` + * platforms it can hold a full Module pointer plus the offset; on such platforms + * it occupies two stack slots, on 64-bit platforms a single one. See the cp + * helpers (make_cp, STORE_CP, LOAD_CP, CP_SIZE_IN_TERMS) in module.h. + */ +typedef uint64_t cp_t; + +#define CP_X_FMT PRIX64 + #if ((UINT32_MAX != 4294967295ULL) || (UINT64_MAX != 18446744073709551615ULL) \ || (INT32_MAX != 2147483647LL) || (INT64_MAX != 9223372036854775807LL)) #error "limits.h or preprocessor is not sane." diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt index a4b0f82b08..aaff3e377d 100644 --- a/tests/erlang_tests/CMakeLists.txt +++ b/tests/erlang_tests/CMakeLists.txt @@ -608,6 +608,7 @@ compile_erlang(test_crypto) compile_erlang(test_code_all_available_loaded) compile_erlang(test_code_load_binary DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/code_load/export_test_module_data.hrl) +compile_erlang(test_many_modules DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/code_load/export_test_module_data.hrl) compile_erlang(test_code_load_abs) compile_erlang(test_code_ensure_loaded) compile_erlang(test_add_avm_pack_binary DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/code_load/code_load_pack_data.hrl) @@ -1172,6 +1173,7 @@ set(erlang_test_beams test_code_all_available_loaded.beam test_code_load_binary.beam + test_many_modules.beam test_code_load_abs.beam test_code_ensure_loaded.beam test_add_avm_pack_binary.beam diff --git a/tests/erlang_tests/test_many_modules.erl b/tests/erlang_tests/test_many_modules.erl new file mode 100644 index 0000000000..c5c536e873 --- /dev/null +++ b/tests/erlang_tests/test_many_modules.erl @@ -0,0 +1,54 @@ +% +% This file is part of AtomVM. +% +% Copyright 2026 Paul Guyot +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +-module(test_many_modules). + +-export([start/0]). + +-include("code_load/export_test_module_data.hrl"). + +-define(MODULE_COUNT, 300). +-define(BASE_NAME, <<"export_test_module">>). + +start() -> + Base = ?EXPORT_TEST_MODULE_DATA, + %% Load MODULE_COUNT (> 256) distinct copies of the same module under + %% different names and call each one. + ok = load_and_call_all(Base, 1), + 0. + +%% 18-char name matching the length of "export_test_module": +%% "export_test_md_" (15 chars) ++ 3-digit zero-padded index (1..999). +mod_name(I) when I < 10 -> + list_to_atom("export_test_md_00" ++ integer_to_list(I)); +mod_name(I) when I < 100 -> + list_to_atom("export_test_md_0" ++ integer_to_list(I)); +mod_name(I) -> + list_to_atom("export_test_md_" ++ integer_to_list(I)). + +load_and_call_all(_Base, I) when I > ?MODULE_COUNT -> + ok; +load_and_call_all(Base, I) -> + Name = mod_name(I), + NameBin = atom_to_binary(Name, latin1), + Bin = binary:replace(Base, ?BASE_NAME, NameBin, [global]), + {module, Name} = code:load_binary(Name, atom_to_list(Name) ++ ".beam", Bin), + 24 = Name:exported_func(4), + load_and_call_all(Base, I + 1). diff --git a/tests/libs/jit/jit_aarch64_tests.erl b/tests/libs/jit/jit_aarch64_tests.erl index 95c64f81b8..259df7682d 100644 --- a/tests/libs/jit/jit_aarch64_tests.erl +++ b/tests/libs/jit/jit_aarch64_tests.erl @@ -2152,10 +2152,17 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: x_reg to reg[x+offset] ?_test(begin - State1 = setelement( - 7, State0, element(7, State0) band (bnot ((1 bsl 8) bor (1 bsl 9))) + Regs0 = element(9, State0), + State2 = setelement( + 9, + State0, + jit_regs:set_masks( + Regs0, + jit_regs:available_regs(Regs0) band + (bnot ((1 bsl 8) bor (1 bsl 9))), + (1 bsl 8) bor (1 bsl 9) + ) ), - State2 = setelement(8, State1, (1 bsl 8) bor (1 bsl 9)), [r8, r9] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, {x_reg, 0}, r8, r9, 1), Stream = ?BACKEND:stream(State3), @@ -2168,10 +2175,17 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: imm to reg[x+offset] ?_test(begin - State1 = setelement( - 7, State0, element(7, State0) band (bnot ((1 bsl 8) bor (1 bsl 9))) + Regs0 = element(9, State0), + State2 = setelement( + 9, + State0, + jit_regs:set_masks( + Regs0, + jit_regs:available_regs(Regs0) band + (bnot ((1 bsl 8) bor (1 bsl 9))), + (1 bsl 8) bor (1 bsl 9) + ) ), - State2 = setelement(8, State1, (1 bsl 8) bor (1 bsl 9)), [r8, r9] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, 42, r8, r9, 1), Stream = ?BACKEND:stream(State3), diff --git a/tests/libs/jit/jit_arm32_tests.erl b/tests/libs/jit/jit_arm32_tests.erl index 5ff9c5749b..4663ae57cd 100644 --- a/tests/libs/jit/jit_arm32_tests.erl +++ b/tests/libs/jit/jit_arm32_tests.erl @@ -400,7 +400,10 @@ move_to_cp_test() -> << " 0: e590a014 ldr sl, [r0, #20]\n" " 4: e59ab000 ldr fp, [sl]\n" - " 8: e580b05c str fp, [r0, #92] @ 0x5c" + " 8: e580b05c str fp, [r0, #92] @ 0x5c\n" + " c: e590a014 ldr sl, [r0, #20]\n" + " 10: e59ab004 ldr fp, [sl, #4]\n" + " 14: e580b060 str fp, [r0, #96] @ 0x60" >>, ?assertStream(arm32, Dump, Stream). @@ -874,9 +877,9 @@ set_bs_test() -> Dump = << " 0: e590b018 ldr fp, [r0, #24]\n" - " 4: e580b064 str fp, [r0, #100] @ 0x64\n" + " 4: e580b068 str fp, [r0, #104] @ 0x68\n" " 8: e3a0a000 mov sl, #0\n" - " c: e580a068 str sl, [r0, #104] @ 0x68" + " c: e580a06c str sl, [r0, #108] @ 0x6c" >>, ?assertStream(arm32, Dump, Stream). @@ -896,51 +899,49 @@ call_or_schedule_next_test() -> Dump = << " 0: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 4: ea000026 b 0xa4\n" + " 4: ea000024 b 0x9c\n" " 8: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" " c: ea000003 b 0x20\n" " 10: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 14: ea000018 b 0x7c\n" + " 14: ea000016 b 0x74\n" " 18: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 1c: ea00001b b 0x90\n" + " 1c: ea000019 b 0x88\n" " 20: e59da000 ldr sl, [sp]\n" " 24: e59ab000 ldr fp, [sl]\n" - " 28: e59bb000 ldr fp, [fp]\n" - " 2c: e1a0bc0b lsl fp, fp, #24\n" - " 30: e3a0ae1e mov sl, #480 @ 0x1e0\n" - " 34: e18bb00a orr fp, fp, sl\n" - " 38: e580b05c str fp, [r0, #92] @ 0x5c\n" - " 3c: e59da000 ldr sl, [sp]\n" - " 40: e59ab008 ldr fp, [sl, #8]\n" - " 44: e25bb001 subs fp, fp, #1\n" - " 48: e58ab008 str fp, [sl, #8]\n" - " 4c: 1a00000a bne 0x7c\n" - " 50: e1a0b00f mov fp, pc\n" - " 54: e3e0a047 mvn sl, #71 @ 0x47\n" - " 58: e08aa00b add sl, sl, fp\n" - " 5c: e59db000 ldr fp, [sp]\n" - " 60: e58ba004 str sl, [fp, #4]\n" - " 64: e592b008 ldr fp, [r2, #8]\n" - " 68: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " 6c: e58db024 str fp, [sp, #36] @ 0x24\n" - " 70: e1a0e007 mov lr, r7\n" - " 74: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" - " 78: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" - " 7c: e592b000 ldr fp, [r2]\n" - " 80: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " 84: e58db024 str fp, [sp, #36] @ 0x24\n" - " 88: e1a0e007 mov lr, r7\n" - " 8c: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" - " 90: e592b004 ldr fp, [r2, #4]\n" - " 94: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " 98: e58db024 str fp, [sp, #36] @ 0x24\n" - " 9c: e1a0e007 mov lr, r7\n" - " a0: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" - " a4: e592b004 ldr fp, [r2, #4]\n" - " a8: e59d7024 ldr r7, [sp, #36] @ 0x24\n" - " ac: e58db024 str fp, [sp, #36] @ 0x24\n" - " b0: e1a0e007 mov lr, r7\n" - " b4: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}" + " 28: e580b060 str fp, [r0, #96] @ 0x60\n" + " 2c: e3a0bd07 mov fp, #448 @ 0x1c0\n" + " 30: e580b05c str fp, [r0, #92] @ 0x5c\n" + " 34: e59da000 ldr sl, [sp]\n" + " 38: e59ab008 ldr fp, [sl, #8]\n" + " 3c: e25bb001 subs fp, fp, #1\n" + " 40: e58ab008 str fp, [sl, #8]\n" + " 44: 1a00000a bne 0x74\n" + " 48: e1a0b00f mov fp, pc\n" + " 4c: e3e0a03f mvn sl, #63 @ 0x3f\n" + " 50: e08aa00b add sl, sl, fp\n" + " 54: e59db000 ldr fp, [sp]\n" + " 58: e58ba004 str sl, [fp, #4]\n" + " 5c: e592b008 ldr fp, [r2, #8]\n" + " 60: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " 64: e58db024 str fp, [sp, #36] @ 0x24\n" + " 68: e1a0e007 mov lr, r7\n" + " 6c: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" + " 70: e92d4ff2 push {r1, r4, r5, r6, r7, r8, r9, sl, fp, lr}\n" + " 74: e592b000 ldr fp, [r2]\n" + " 78: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " 7c: e58db024 str fp, [sp, #36] @ 0x24\n" + " 80: e1a0e007 mov lr, r7\n" + " 84: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" + " 88: e592b004 ldr fp, [r2, #4]\n" + " 8c: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " 90: e58db024 str fp, [sp, #36] @ 0x24\n" + " 94: e1a0e007 mov lr, r7\n" + " 98: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}\n" + " 9c: e592b004 ldr fp, [r2, #4]\n" + " a0: e59d7024 ldr r7, [sp, #36] @ 0x24\n" + " a4: e58db024 str fp, [sp, #36] @ 0x24\n" + " a8: e1a0e007 mov lr, r7\n" + " ac: e8bd8ff2 pop {r1, r4, r5, r6, r7, r8, r9, sl, fp, pc}" >>, ?assertStream(arm32, Dump, Stream). diff --git a/tests/libs/jit/jit_armv6m_tests.erl b/tests/libs/jit/jit_armv6m_tests.erl index 408f908c16..fb3c099e46 100644 --- a/tests/libs/jit/jit_armv6m_tests.erl +++ b/tests/libs/jit/jit_armv6m_tests.erl @@ -443,7 +443,10 @@ move_to_cp_test() -> << " 0: 6946 ldr r6, [r0, #20]\n" " 2: 6837 ldr r7, [r6, #0]\n" - " 4: 65c7 str r7, [r0, #92] ; 0x5c" + " 4: 65c7 str r7, [r0, #92] @ 0x5c\n" + " 6: 6946 ldr r6, [r0, #20]\n" + " 8: 6877 ldr r7, [r6, #4]\n" + " a: 6607 str r7, [r0, #96] @ 0x60" >>, ?assertStream(arm, Dump, Stream). @@ -2341,7 +2344,7 @@ call_ext_test() -> " 4: 3f01 subs r7, #1\n" " 6: 60b7 str r7, [r6, #8]\n" " 8: d109 bne.n 0x1e\n" - " a: a704 add r7, pc, #16 ; (adr r7, 0x1c)\n" + " a: a704 add r7, pc, #16 @ (adr r7, 0x1c)\n" " c: 3701 adds r7, #1\n" " e: 6077 str r7, [r6, #4]\n" " 10: 6897 ldr r7, [r2, #8]\n" @@ -2349,30 +2352,28 @@ call_ext_test() -> " 14: 9705 str r7, [sp, #20]\n" " 16: 46b6 mov lr, r6\n" " 18: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 1a: 46c0 nop ; (mov r8, r8)\n" + " 1a: 46c0 nop @ (mov r8, r8)\n" " 1c: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 1e: 9e00 ldr r6, [sp, #0]\n" " 20: 6837 ldr r7, [r6, #0]\n" - " 22: 683f ldr r7, [r7, #0]\n" - " 24: 063f lsls r7, r7, #24\n" - " 26: 4e07 ldr r6, [pc, #28] ; (0x44)\n" - " 28: 4337 orrs r7, r6\n" - " 2a: 65c7 str r7, [r0, #92] ; 0x5c\n" - " 2c: 6917 ldr r7, [r2, #16]\n" - " 2e: b082 sub sp, #8\n" - " 30: 2601 movs r6, #1\n" - " 32: 4276 negs r6, r6\n" - " 34: 9600 str r6, [sp, #0]\n" - " 36: 9902 ldr r1, [sp, #8]\n" - " 38: 2202 movs r2, #2\n" - " 3a: 2305 movs r3, #5\n" - " 3c: 47b8 blx r7\n" - " 3e: b002 add sp, #8\n" - " 40: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 22: 6607 str r7, [r0, #96] @ 0x60\n" + " 24: 4f06 ldr r7, [pc, #24] @ (0x40)\n" + " 26: 65c7 str r7, [r0, #92] @ 0x5c\n" + " 28: 6917 ldr r7, [r2, #16]\n" + " 2a: b082 sub sp, #8\n" + " 2c: 2601 movs r6, #1\n" + " 2e: 4276 negs r6, r6\n" + " 30: 9600 str r6, [sp, #0]\n" + " 32: 9902 ldr r1, [sp, #8]\n" + " 34: 2202 movs r2, #2\n" + " 36: 2305 movs r3, #5\n" + " 38: 47b8 blx r7\n" + " 3a: b002 add sp, #8\n" + " 3c: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 3e: 0000 movs r0, r0\n" + " 40: 0110 lsls r0, r2, #4\n" " 42: 0000 movs r0, r0\n" - " 44: 0120 lsls r0, r4, #4\n" - " 46: 0000 movs r0, r0\n" - " 48: b5f2 push {r1, r4, r5, r6, r7, lr}" + " 44: b5f2 push {r1, r4, r5, r6, r7, lr}" >>, ?assertStream(arm, Dump, Stream). @@ -2411,7 +2412,7 @@ call_fun_test() -> " 4: 3f01 subs r7, #1\n" " 6: 60b7 str r7, [r6, #8]\n" " 8: d109 bne.n 0x1e\n" - " a: a704 add r7, pc, #16 ; (adr r7, 0x1c)\n" + " a: a704 add r7, pc, #16 @ (adr r7, 0x1c)\n" " c: 3701 adds r7, #1\n" " e: 6077 str r7, [r6, #4]\n" " 10: 6897 ldr r7, [r2, #8]\n" @@ -2419,7 +2420,7 @@ call_fun_test() -> " 14: 9705 str r7, [sp, #20]\n" " 16: 46b6 mov lr, r6\n" " 18: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 1a: 46c0 nop ; (mov r8, r8)\n" + " 1a: 46c0 nop @ (mov r8, r8)\n" " 1c: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 1e: 6987 ldr r7, [r0, #24]\n" " 20: 463e mov r6, r7\n" @@ -2428,13 +2429,13 @@ call_fun_test() -> " 26: 4025 ands r5, r4\n" " 28: 2d02 cmp r5, #2\n" " 2a: d009 beq.n 0x40\n" - " 2c: 6cd7 ldr r7, [r2, #76] ; 0x4c\n" + " 2c: 6cd7 ldr r7, [r2, #76] @ 0x4c\n" " 2e: b082 sub sp, #8\n" " 30: 9600 str r6, [sp, #0]\n" " 32: 9902 ldr r1, [sp, #8]\n" - " 34: 222e movs r2, #46 ; 0x2e\n" - " 36: 23ff movs r3, #255 ; 0xff\n" - " 38: 338c adds r3, #140 ; 0x8c\n" + " 34: 222e movs r2, #46 @ 0x2e\n" + " 36: 23ff movs r3, #255 @ 0xff\n" + " 38: 338c adds r3, #140 @ 0x8c\n" " 3a: 47b8 blx r7\n" " 3c: b002 add sp, #8\n" " 3e: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" @@ -2442,39 +2443,37 @@ call_fun_test() -> " 42: 43ae bics r6, r5\n" " 44: 6836 ldr r6, [r6, #0]\n" " 46: 4635 mov r5, r6\n" - " 48: 243f movs r4, #63 ; 0x3f\n" + " 48: 243f movs r4, #63 @ 0x3f\n" " 4a: 4025 ands r5, r4\n" " 4c: 2d14 cmp r5, #20\n" " 4e: d009 beq.n 0x64\n" - " 50: 6cd7 ldr r7, [r2, #76] ; 0x4c\n" + " 50: 6cd7 ldr r7, [r2, #76] @ 0x4c\n" " 52: b082 sub sp, #8\n" " 54: 9600 str r6, [sp, #0]\n" " 56: 9902 ldr r1, [sp, #8]\n" - " 58: 2252 movs r2, #82 ; 0x52\n" - " 5a: 23ff movs r3, #255 ; 0xff\n" - " 5c: 338c adds r3, #140 ; 0x8c\n" + " 58: 2252 movs r2, #82 @ 0x52\n" + " 5a: 23ff movs r3, #255 @ 0xff\n" + " 5c: 338c adds r3, #140 @ 0x8c\n" " 5e: 47b8 blx r7\n" " 60: b002 add sp, #8\n" " 62: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" " 64: 9d00 ldr r5, [sp, #0]\n" " 66: 682e ldr r6, [r5, #0]\n" - " 68: 6836 ldr r6, [r6, #0]\n" - " 6a: 0636 lsls r6, r6, #24\n" - " 6c: 4d05 ldr r5, [pc, #20] ; (0x84)\n" - " 6e: 432e orrs r6, r5\n" - " 70: 65c6 str r6, [r0, #92] ; 0x5c\n" - " 72: 2680 movs r6, #128 ; 0x80\n" - " 74: 5996 ldr r6, [r2, r6]\n" - " 76: 463a mov r2, r7\n" - " 78: 2300 movs r3, #0\n" - " 7a: 9f05 ldr r7, [sp, #20]\n" - " 7c: 9605 str r6, [sp, #20]\n" - " 7e: 46be mov lr, r7\n" - " 80: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 68: 6606 str r6, [r0, #96] @ 0x60\n" + " 6a: 4e05 ldr r6, [pc, #20] @ (0x80)\n" + " 6c: 65c6 str r6, [r0, #92] @ 0x5c\n" + " 6e: 2680 movs r6, #128 @ 0x80\n" + " 70: 5996 ldr r6, [r2, r6]\n" + " 72: 463a mov r2, r7\n" + " 74: 2300 movs r3, #0\n" + " 76: 9f05 ldr r7, [sp, #20]\n" + " 78: 9605 str r6, [sp, #20]\n" + " 7a: 46be mov lr, r7\n" + " 7c: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 7e: 0000 movs r0, r0\n" + " 80: 0210 lsls r0, r2, #8\n" " 82: 0000 movs r0, r0\n" - " 84: 0220 lsls r0, r4, #8\n" - " 86: 0000 movs r0, r0\n" - " 88: b5f2 push {r1, r4, r5, r6, r7, lr}" + " 84: b5f2 push {r1, r4, r5, r6, r7, lr}" >>, ?assertStream(arm, Dump, Stream). @@ -3034,10 +3033,16 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: x_reg to reg[x+offset] ?_test(begin - State1 = setelement( - 7, State0, element(7, State0) band (bnot ((1 bsl 3) bor (1 bsl 4))) + %% State record now stores masks inside the regs field + %% (position 10). Update the masks via jit_regs:set_masks. + Regs0 = element(10, State0), + UsedMask = (1 bsl 3) bor (1 bsl 4), + Regs1 = jit_regs:set_masks( + Regs0, + jit_regs:available_regs(Regs0) band (bnot UsedMask), + UsedMask ), - State2 = setelement(8, State1, (1 bsl 3) bor (1 bsl 4)), + State2 = setelement(10, State0, Regs1), [r4, r3] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, {x_reg, 0}, r3, r4, 1), Stream = ?BACKEND:stream(State3), @@ -3051,10 +3056,14 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: imm to reg[x+offset] ?_test(begin - State1 = setelement( - 7, State0, element(7, State0) band (bnot ((1 bsl 3) bor (1 bsl 4))) + Regs0 = element(10, State0), + UsedMask = (1 bsl 3) bor (1 bsl 4), + Regs1 = jit_regs:set_masks( + Regs0, + jit_regs:available_regs(Regs0) band (bnot UsedMask), + UsedMask ), - State2 = setelement(8, State1, (1 bsl 3) bor (1 bsl 4)), + State2 = setelement(10, State0, Regs1), [r4, r3] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, 42, r3, r4, 1), Stream = ?BACKEND:stream(State3), @@ -3207,7 +3216,7 @@ move_to_native_register_test_() -> Stream = ?BACKEND:stream(State2), Dump = << " 0: 6987 ldr r7, [r0, #24]\n" - " 2: 6e06 ldr r6, [r0, #96] ; 0x60\n" + " 2: 6e46 ldr r6, [r0, #100] @ 0x64\n" " 4: 687d ldr r5, [r7, #4]\n" " 6: 61b5 str r5, [r6, #24]\n" " 8: 68bd ldr r5, [r7, #8]\n" @@ -3956,51 +3965,46 @@ add_beam_test() -> Stream = ?BACKEND:stream(State15), Dump = << - % jump table - " 0: 4b01 ldr r3, [pc, #4] ; (0x8)\n" + " 0: 4b01 ldr r3, [pc, #4] @ (0x8)\n" " 2: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 4: 449f add pc, r3\n" - " 6: 46c0 nop ; (mov r8, r8)\n" - " 8: 00d9 lsls r1, r3, #3\n" + " 6: 46c0 nop @ (mov r8, r8)\n" + " 8: 00d5 lsls r5, r2, #3\n" " a: 0000 movs r0, r0\n" - " c: 4b01 ldr r3, [pc, #4] ; (0x14)\n" + " c: 4b01 ldr r3, [pc, #4] @ (0x14)\n" " e: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 10: 449f add pc, r3\n" - " 12: 46c0 nop ; (mov r8, r8)\n" + " 12: 46c0 nop @ (mov r8, r8)\n" " 14: 001d movs r5, r3\n" " 16: 0000 movs r0, r0\n" - " 18: 4b01 ldr r3, [pc, #4] ; (0x20)\n" + " 18: 4b01 ldr r3, [pc, #4] @ (0x20)\n" " 1a: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 1c: 449f add pc, r3\n" - " 1e: 46c0 nop ; (mov r8, r8)\n" + " 1e: 46c0 nop @ (mov r8, r8)\n" " 20: 0045 lsls r5, r0, #1\n" " 22: 0000 movs r0, r0\n" - " 24: 4b01 ldr r3, [pc, #4] ; (0x2c)\n" + " 24: 4b01 ldr r3, [pc, #4] @ (0x2c)\n" " 26: b5f2 push {r1, r4, r5, r6, r7, lr}\n" " 28: 449f add pc, r3\n" - " 2a: 46c0 nop ; (mov r8, r8)\n" - " 2c: 00a9 lsls r1, r5, #2\n" + " 2a: 46c0 nop @ (mov r8, r8)\n" + " 2c: 00a5 lsls r5, r4, #2\n" " 2e: 0000 movs r0, r0\n" - % label 1 - % {move,{integer,9},{x,1}}. - " 30: 279f movs r7, #159 ; 0x9f\n" + " 30: 279f movs r7, #159 @ 0x9f\n" " 32: 61c7 str r7, [r0, #28]\n" - % {move,{integer,8},{x,0}} - " 34: 278f movs r7, #143 ; 0x8f\n" + " 34: 278f movs r7, #143 @ 0x8f\n" " 36: 6187 str r7, [r0, #24]\n" - % {call_only,2,{f,2}}. " 38: 9e00 ldr r6, [sp, #0]\n" " 3a: 68b7 ldr r7, [r6, #8]\n" " 3c: 3f01 subs r7, #1\n" " 3e: 60b7 str r7, [r6, #8]\n" " 40: d004 beq.n 0x4c\n" " 42: e00f b.n 0x64\n" - " 44: 46c0 nop ; (mov r8, r8)\n" - " 46: 46c0 nop ; (mov r8, r8)\n" - " 48: 46c0 nop ; (mov r8, r8)\n" - " 4a: 46c0 nop ; (mov r8, r8)\n" - " 4c: a700 add r7, pc, #0 ; (adr r7, 0x50)\n" - " 4e: 2637 movs r6, #55 ; 0x37\n" + " 44: 46c0 nop @ (mov r8, r8)\n" + " 46: 46c0 nop @ (mov r8, r8)\n" + " 48: 46c0 nop @ (mov r8, r8)\n" + " 4a: 46c0 nop @ (mov r8, r8)\n" + " 4c: a700 add r7, pc, #0 @ (adr r7, 0x50)\n" + " 4e: 2637 movs r6, #55 @ 0x37\n" " 50: 4276 negs r6, r6\n" " 52: 19f6 adds r6, r6, r7\n" " 54: 9f00 ldr r7, [sp, #0]\n" @@ -4010,9 +4014,7 @@ add_beam_test() -> " 5c: 9705 str r7, [sp, #20]\n" " 5e: 46b6 mov lr, r6\n" " 60: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 62: 46c0 nop ; (mov r8, r8)\n" - % label 2 - % {allocate,1,1}. + " 62: 46c0 nop @ (mov r8, r8)\n" " 64: 6957 ldr r7, [r2, #20]\n" " 66: b405 push {r0, r2}\n" " 68: b082 sub sp, #8\n" @@ -4028,65 +4030,56 @@ add_beam_test() -> " 7c: 07fe lsls r6, r7, #31\n" " 7e: d405 bmi.n 0x8c\n" " 80: 6997 ldr r7, [r2, #24]\n" - " 82: 2282 movs r2, #130 ; 0x82\n" + " 82: 2282 movs r2, #130 @ 0x82\n" " 84: 9e05 ldr r6, [sp, #20]\n" " 86: 9705 str r7, [sp, #20]\n" " 88: 46b6 mov lr, r6\n" " 8a: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - % {init_yregs,{list,[{y,0}]}}. - %% move_to_vm_register(State8, ?TERM_NIL, {y_reg, 0}), - " 8c: 263b movs r6, #59 ; 0x3b\n" + " 8c: 263b movs r6, #59 @ 0x3b\n" " 8e: 6947 ldr r7, [r0, #20]\n" " 90: 603e str r6, [r7, #0]\n" - % {call,1,{f,3}} - %% call_or_schedule_next(State9, 3), " 92: 9e00 ldr r6, [sp, #0]\n" " 94: 6837 ldr r7, [r6, #0]\n" - " 96: 683f ldr r7, [r7, #0]\n" - " 98: 063f lsls r7, r7, #24\n" - " 9a: 4e0c ldr r6, [pc, #48] ; (0xcc)\n" - " 9c: 4337 orrs r7, r6\n" - " 9e: 65c7 str r7, [r0, #92] ; 0x5c\n" - " a0: 9e00 ldr r6, [sp, #0]\n" - " a2: 68b7 ldr r7, [r6, #8]\n" - " a4: 3f01 subs r7, #1\n" - " a6: 60b7 str r7, [r6, #8]\n" - " a8: d004 beq.n 0xb4\n" - " aa: e013 b.n 0xd4\n" - " ac: 46c0 nop ; (mov r8, r8)\n" - " ae: 46c0 nop ; (mov r8, r8)\n" - " b0: 46c0 nop ; (mov r8, r8)\n" - " b2: 46c0 nop ; (mov r8, r8)\n" - " b4: a700 add r7, pc, #0 ; (adr r7, 0xb8)\n" - " b6: 2693 movs r6, #147 ; 0x93\n" - " b8: 4276 negs r6, r6\n" - " ba: 19f6 adds r6, r6, r7\n" - " bc: 9f00 ldr r7, [sp, #0]\n" - " be: 607e str r6, [r7, #4]\n" - " c0: 6897 ldr r7, [r2, #8]\n" - " c2: 9e05 ldr r6, [sp, #20]\n" - " c4: 9705 str r7, [sp, #20]\n" - " c6: 46b6 mov lr, r6\n" - " c8: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 96: 6607 str r7, [r0, #96] @ 0x60\n" + " 98: 4f0b ldr r7, [pc, #44] @ (0xc8)\n" + " 9a: 65c7 str r7, [r0, #92] @ 0x5c\n" + " 9c: 9e00 ldr r6, [sp, #0]\n" + " 9e: 68b7 ldr r7, [r6, #8]\n" + " a0: 3f01 subs r7, #1\n" + " a2: 60b7 str r7, [r6, #8]\n" + " a4: d004 beq.n 0xb0\n" + " a6: e013 b.n 0xd0\n" + " a8: 46c0 nop @ (mov r8, r8)\n" + " aa: 46c0 nop @ (mov r8, r8)\n" + " ac: 46c0 nop @ (mov r8, r8)\n" + " ae: 46c0 nop @ (mov r8, r8)\n" + " b0: a700 add r7, pc, #0 @ (adr r7, 0xb4)\n" + " b2: 268f movs r6, #143 @ 0x8f\n" + " b4: 4276 negs r6, r6\n" + " b6: 19f6 adds r6, r6, r7\n" + " b8: 9f00 ldr r7, [sp, #0]\n" + " ba: 607e str r6, [r7, #4]\n" + " bc: 6897 ldr r7, [r2, #8]\n" + " be: 9e05 ldr r6, [sp, #20]\n" + " c0: 9705 str r7, [sp, #20]\n" + " c2: 46b6 mov lr, r6\n" + " c4: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " c6: 0000 movs r0, r0\n" + " c8: 0330 lsls r0, r6, #12\n" " ca: 0000 movs r0, r0\n" - " cc: 0340 lsls r0, r0, #13\n" - " ce: 0000 movs r0, r0\n" - %% (continuation) - " d0: b5f2 push {r1, r4, r5, r6, r7, lr}\n" - " d2: 46c0 nop ; (mov r8, r8)\n" - % label 3 - " d4: 6857 ldr r7, [r2, #4]\n" - " d6: 9e05 ldr r6, [sp, #20]\n" - " d8: 9705 str r7, [sp, #20]\n" - " da: 46b6 mov lr, r6\n" - " dc: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " de: 46c0 nop ; (mov r8, r8)\n" - % label 0 - " e0: 6857 ldr r7, [r2, #4]\n" - " e2: 9e05 ldr r6, [sp, #20]\n" - " e4: 9705 str r7, [sp, #20]\n" - " e6: 46b6 mov lr, r6\n" - " e8: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " cc: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " ce: 46c0 nop @ (mov r8, r8)\n" + " d0: 6857 ldr r7, [r2, #4]\n" + " d2: 9e05 ldr r6, [sp, #20]\n" + " d4: 9705 str r7, [sp, #20]\n" + " d6: 46b6 mov lr, r6\n" + " d8: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " da: 46c0 nop @ (mov r8, r8)\n" + " dc: 6857 ldr r7, [r2, #4]\n" + " de: 9e05 ldr r6, [sp, #20]\n" + " e0: 9705 str r7, [sp, #20]\n" + " e2: 46b6 mov lr, r6\n" + " e4: bdf2 pop {r1, r4, r5, r6, r7, pc}" >>, ?assertStream(arm, Dump, Stream). diff --git a/tests/libs/jit/jit_regs_tests.erl b/tests/libs/jit/jit_regs_tests.erl index 059f62927e..a7f1cfa395 100644 --- a/tests/libs/jit/jit_regs_tests.erl +++ b/tests/libs/jit/jit_regs_tests.erl @@ -150,3 +150,100 @@ vm_dest_to_contents_test() -> ?assertEqual({y_reg, 5}, jit_regs:vm_dest_to_contents({y_reg, 5}, MaxReg)), ?assertEqual(unknown, jit_regs:vm_dest_to_contents({fp_reg, 0}, MaxReg)), ?assertEqual(unknown, jit_regs:vm_dest_to_contents({ptr, rax}, MaxReg)). + +%% Native-register bookkeeping (masks). A small fixture mapping four +%% registers to single bits, used to exercise the shared bit-law helpers +%% independently of any backend. +-define(R0_BIT, 1). +-define(R1_BIT, 2). +-define(R2_BIT, 4). +-define(R3_BIT, 8). +-define(ALL_REGS, (?R0_BIT bor ?R1_BIT bor ?R2_BIT bor ?R3_BIT)). + +reg_bit(r0) -> ?R0_BIT; +reg_bit(r1) -> ?R1_BIT; +reg_bit(r2) -> ?R2_BIT; +reg_bit(r3) -> ?R3_BIT. + +new_masks_test() -> + Regs = jit_regs:new(?ALL_REGS, 0), + ?assertEqual(?ALL_REGS, jit_regs:available_regs(Regs)), + ?assertEqual(0, jit_regs:used_regs(Regs)), + %% new/0 starts both masks empty. + Empty = jit_regs:new(), + ?assertEqual(0, jit_regs:available_regs(Empty)), + ?assertEqual(0, jit_regs:used_regs(Empty)). + +set_masks_test() -> + Regs0 = jit_regs:new(?ALL_REGS, 0), + Regs1 = jit_regs:set_masks(Regs0, ?R2_BIT bor ?R3_BIT, ?R0_BIT bor ?R1_BIT), + ?assertEqual(?R2_BIT bor ?R3_BIT, jit_regs:available_regs(Regs1)), + ?assertEqual(?R0_BIT bor ?R1_BIT, jit_regs:used_regs(Regs1)), + Regs2 = jit_regs:set_available_regs(Regs1, ?ALL_REGS), + ?assertEqual(?ALL_REGS, jit_regs:available_regs(Regs2)), + %% set_available_regs leaves used_regs untouched. + ?assertEqual(?R0_BIT bor ?R1_BIT, jit_regs:used_regs(Regs2)). + +alloc_free_reg_test() -> + Regs0 = jit_regs:new(?ALL_REGS, 0), + Regs1 = jit_regs:alloc_reg(Regs0, ?R0_BIT), + ?assertEqual(?ALL_REGS band (bnot ?R0_BIT), jit_regs:available_regs(Regs1)), + ?assertEqual(?R0_BIT, jit_regs:used_regs(Regs1)), + %% free_reg is the inverse of alloc_reg: round-trips back to the start. + Regs2 = jit_regs:free_reg(Regs1, ?R0_BIT), + ?assertEqual(?ALL_REGS, jit_regs:available_regs(Regs2)), + ?assertEqual(0, jit_regs:used_regs(Regs2)). + +alloc_reg_multiple_bits_test() -> + %% Bit may carry more than one register (e.g. armv6m allocates a pair). + Regs0 = jit_regs:new(?ALL_REGS, 0), + Regs1 = jit_regs:alloc_reg(Regs0, ?R0_BIT bor ?R2_BIT), + ?assertEqual(?R1_BIT bor ?R3_BIT, jit_regs:available_regs(Regs1)), + ?assertEqual(?R0_BIT bor ?R2_BIT, jit_regs:used_regs(Regs1)), + Regs2 = jit_regs:free_reg(Regs1, ?R0_BIT bor ?R2_BIT), + ?assertEqual(?ALL_REGS, jit_regs:available_regs(Regs2)), + ?assertEqual(0, jit_regs:used_regs(Regs2)). + +merge_masks_test() -> + %% Branch A used r0; branch B used r1. After the merge both are used and + %% available is the intersection, constrained by ?ALL_REGS. + RegsA = jit_regs:alloc_reg(jit_regs:new(?ALL_REGS, 0), ?R0_BIT), + RegsB = jit_regs:alloc_reg(jit_regs:new(?ALL_REGS, 0), ?R1_BIT), + Merged = jit_regs:merge(RegsA, RegsB, ?ALL_REGS), + ?assertEqual(?R0_BIT bor ?R1_BIT, jit_regs:used_regs(Merged)), + ?assertEqual(?R2_BIT bor ?R3_BIT, jit_regs:available_regs(Merged)). + +merge_masks_constrained_test() -> + %% Bits outside AllRegsMask are dropped from both masks. + RegsA = jit_regs:new(?ALL_REGS bor 16#10, ?R0_BIT bor 16#20), + RegsB = jit_regs:new(?ALL_REGS bor 16#10, ?R1_BIT), + Merged = jit_regs:merge(RegsA, RegsB, ?ALL_REGS), + ?assertEqual(?R0_BIT bor ?R1_BIT, jit_regs:used_regs(Merged)), + ?assertEqual(?ALL_REGS, jit_regs:available_regs(Merged)). + +first_set_test() -> + F = fun reg_bit/1, + Order = [r0, r1, r2, r3], + %% Follows the list order, not the numeric bit order. + ?assertEqual(r1, jit_regs:first_set(?R1_BIT bor ?R3_BIT, Order, F)), + ?assertEqual(r0, jit_regs:first_set(?ALL_REGS, Order, F)), + ?assertEqual(r3, jit_regs:first_set(?R3_BIT, Order, F)), + %% A different list order changes the winner for the same mask. + ?assertEqual(r3, jit_regs:first_set(?R1_BIT bor ?R3_BIT, [r3, r2, r1, r0], F)). + +first_set_no_match_crashes_test() -> + F = fun reg_bit/1, + %% Documented contract: crashes when no listed register's bit is set. + ?assertError(function_clause, jit_regs:first_set(0, [r0, r1], F)), + %% Non-empty mask but no bit in the list also crashes. + ?assertError(function_clause, jit_regs:first_set(16#10, [r0, r1, r2, r3], F)). + +mask_to_reg_list_test() -> + F = fun reg_bit/1, + Order = [r0, r1, r2, r3], + ?assertEqual([r0, r2], jit_regs:mask_to_reg_list(?R0_BIT bor ?R2_BIT, Order, F)), + ?assertEqual([r0, r1, r2, r3], jit_regs:mask_to_reg_list(?ALL_REGS, Order, F)), + ?assertEqual([], jit_regs:mask_to_reg_list(0, Order, F)), + %% Order follows the list argument, and bits not in the list are ignored. + ?assertEqual([r3, r1], jit_regs:mask_to_reg_list(?ALL_REGS, [r3, r1], F)), + ?assertEqual([], jit_regs:mask_to_reg_list(16#10, Order, F)). diff --git a/tests/libs/jit/jit_riscv32_tests.erl b/tests/libs/jit/jit_riscv32_tests.erl index b777f2d807..a86aa6ba3c 100644 --- a/tests/libs/jit/jit_riscv32_tests.erl +++ b/tests/libs/jit/jit_riscv32_tests.erl @@ -446,9 +446,12 @@ move_to_cp_test() -> Stream = ?BACKEND:stream(State1), Dump = << - " 0: 01452f03 lw t5,20(a0)\n" - " 4: 000f2f83 lw t6,0(t5)\n" - " 8: 05f52e23 sw t6,92(a0)" + " 0: 01452f03 lw t5,20(a0)\n" + " 4: 000f2f83 lw t6,0(t5)\n" + " 8: 05f52e23 sw t6,92(a0)\n" + " c: 01452f03 lw t5,20(a0)\n" + " 10: 004f2f83 lw t6,4(t5)\n" + " 14: 07f52023 sw t6,96(a0)" >>, ?assertStream(riscv32, Dump, Stream). @@ -1998,28 +2001,26 @@ call_ext_test() -> Stream = ?BACKEND:stream(State2), Dump = << - " 0: 0085af83 lw t6,8(a1)\n" - " 4: 1ffd addi t6,t6,-1\n" - " 6: 01f5a423 sw t6,8(a1)\n" - " a: 000f9b63 bnez t6,0x20\n" - " e: 00000f97 auipc t6,0x0\n" - " 12: 0fc9 addi t6,t6,18 # 0x20\n" - " 14: 0001 nop\n" - " 16: 01f5a223 sw t6,4(a1)\n" - " 1a: 00862f83 lw t6,8(a2)\n" - " 1e: 8f82 jr t6\n" - " 20: 0005af03 lw t5,0(a1)\n" - " 24: 000f2f03 lw t5,0(t5)\n" - " 28: 0f62 slli t5,t5,0x18\n" - " 2a: 11800f93 li t6,280\n" - " 2e: 00000013 nop\n" - " 32: 01ff6f33 or t5,t5,t6\n" - " 36: 05e52e23 sw t5,92(a0)\n" - " 3a: 01062f83 lw t6,16(a2)\n" - " 3e: 4609 li a2,2\n" - " 40: 4695 li a3,5\n" - " 42: 577d li a4,-1\n" - " 44: 8f82 jr t6" + " 0: 0085af83 lw t6,8(a1)\n" + " 4: 1ffd addi t6,t6,-1\n" + " 6: 01f5a423 sw t6,8(a1)\n" + " a: 000f9b63 bnez t6,0x20\n" + " e: 00000f97 auipc t6,0x0\n" + " 12: 0fc9 addi t6,t6,18 # 0x20\n" + " 14: 0001 nop\n" + " 16: 01f5a223 sw t6,4(a1)\n" + " 1a: 00862f83 lw t6,8(a2)\n" + " 1e: 8f82 jr t6\n" + " 20: 0005af83 lw t6,0(a1)\n" + " 24: 07f52023 sw t6,96(a0)\n" + " 28: 10000f93 li t6,256\n" + " 2c: 00000013 nop\n" + " 30: 05f52e23 sw t6,92(a0)\n" + " 34: 01062f83 lw t6,16(a2)\n" + " 38: 4609 li a2,2\n" + " 3a: 4695 li a3,5\n" + " 3c: 577d li a4,-1\n" + " 3e: 8f82 jr t6" >>, ?assertStream(riscv32, Dump, Stream). @@ -2054,51 +2055,49 @@ call_fun_test() -> Stream = ?BACKEND:stream(State9), Dump = << - " 0: 0085af83 lw t6,8(a1)\n" - " 4: 1ffd addi t6,t6,-1\n" - " 6: 01f5a423 sw t6,8(a1)\n" - " a: 000f9b63 bnez t6,0x20\n" - " e: 00000f97 auipc t6,0x0\n" - " 12: 0fc9 addi t6,t6,18 # 0x20\n" - " 14: 0001 nop\n" - " 16: 01f5a223 sw t6,4(a1)\n" - " 1a: 00862f83 lw t6,8(a2)\n" - " 1e: 8f82 jr t6\n" - " 20: 01852f83 lw t6,24(a0)\n" - " 24: 8f7e mv t5,t6\n" - " 26: 8efa mv t4,t5\n" - " 28: 003efe93 andi t4,t4,3\n" - " 2c: 4e09 li t3,2\n" - " 2e: 01ce8a63 beq t4,t3,0x42\n" - " 32: 04c62f83 lw t6,76(a2)\n" - " 36: 03600613 li a2,54\n" - " 3a: 18b00693 li a3,395\n" - " 3e: 877a mv a4,t5\n" - " 40: 8f82 jr t6\n" - " 42: ffcf7f13 andi t5,t5,-4\n" - " 46: 000f2f03 lw t5,0(t5)\n" - " 4a: 8efa mv t4,t5\n" - " 4c: 03fefe93 andi t4,t4,63\n" - " 50: 4e51 li t3,20\n" - " 52: 01ce8a63 beq t4,t3,0x66\n" - " 56: 04c62f83 lw t6,76(a2)\n" - " 5a: 05a00613 li a2,90\n" - " 5e: 18b00693 li a3,395\n" - " 62: 877a mv a4,t5\n" - " 64: 8f82 jr t6\n" - " 66: 0005ae83 lw t4,0(a1)\n" - " 6a: 000eae83 lw t4,0(t4)\n" - " 6e: 0ee2 slli t4,t4,0x18\n" - " 70: 24000f13 li t5,576\n" - " 74: 00000013 nop\n" - " 78: 01eeeeb3 or t4,t4,t5\n" - " 7c: 05d52e23 sw t4,92(a0)\n" - " 80: 08000f13 li t5,128\n" - " 84: 9f32 add t5,t5,a2\n" - " 86: 000f2f03 lw t5,0(t5)\n" - " 8a: 867e mv a2,t6\n" - " 8c: 4681 li a3,0\n" - " 8e: 8f02 jr t5" + " 0: 0085af83 lw t6,8(a1)\n" + " 4: 1ffd addi t6,t6,-1\n" + " 6: 01f5a423 sw t6,8(a1)\n" + " a: 000f9b63 bnez t6,0x20\n" + " e: 00000f97 auipc t6,0x0\n" + " 12: 0fc9 addi t6,t6,18 # 0x20\n" + " 14: 0001 nop\n" + " 16: 01f5a223 sw t6,4(a1)\n" + " 1a: 00862f83 lw t6,8(a2)\n" + " 1e: 8f82 jr t6\n" + " 20: 01852f83 lw t6,24(a0)\n" + " 24: 8f7e mv t5,t6\n" + " 26: 8efa mv t4,t5\n" + " 28: 003efe93 andi t4,t4,3\n" + " 2c: 4e09 li t3,2\n" + " 2e: 01ce8a63 beq t4,t3,0x42\n" + " 32: 04c62f83 lw t6,76(a2)\n" + " 36: 03600613 li a2,54\n" + " 3a: 18b00693 li a3,395\n" + " 3e: 877a mv a4,t5\n" + " 40: 8f82 jr t6\n" + " 42: ffcf7f13 andi t5,t5,-4\n" + " 46: 000f2f03 lw t5,0(t5)\n" + " 4a: 8efa mv t4,t5\n" + " 4c: 03fefe93 andi t4,t4,63\n" + " 50: 4e51 li t3,20\n" + " 52: 01ce8a63 beq t4,t3,0x66\n" + " 56: 04c62f83 lw t6,76(a2)\n" + " 5a: 05a00613 li a2,90\n" + " 5e: 18b00693 li a3,395\n" + " 62: 877a mv a4,t5\n" + " 64: 8f82 jr t6\n" + " 66: 0005af03 lw t5,0(a1)\n" + " 6a: 07e52023 sw t5,96(a0)\n" + " 6e: 22800f13 li t5,552\n" + " 72: 00000013 nop\n" + " 76: 05e52e23 sw t5,92(a0)\n" + " 7a: 08000f13 li t5,128\n" + " 7e: 9f32 add t5,t5,a2\n" + " 80: 000f2f03 lw t5,0(t5)\n" + " 84: 867e mv a2,t6\n" + " 86: 4681 li a3,0\n" + " 88: 8f02 jr t5" >>, ?assertStream(riscv32, Dump, Stream). @@ -2583,8 +2582,10 @@ move_to_array_element_test_() -> ?_test(begin %% a3 = bit 3 = 8, t3 = bit 11 = 2048 %% AVAILABLE_REGS_MASK = 16#7F00, remove t3 => 16#7700 - State1 = setelement(7, State0, 16#7700), - State2 = setelement(8, State1, 8 bor 2048), + Regs0 = element(9, State0), + State2 = setelement( + 9, State0, jit_regs:set_masks(Regs0, 16#7700, 8 bor 2048) + ), [t3, a3] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, {x_reg, 0}, a3, t3, 1), Stream = ?BACKEND:stream(State3), @@ -2601,8 +2602,10 @@ move_to_array_element_test_() -> ?_test(begin %% a3 = bit 3 = 8, t3 = bit 11 = 2048 %% AVAILABLE_REGS_MASK = 16#7F00, remove t3 => 16#7700 - State1 = setelement(7, State0, 16#7700), - State2 = setelement(8, State1, 8 bor 2048), + Regs0 = element(9, State0), + State2 = setelement( + 9, State0, jit_regs:set_masks(Regs0, 16#7700, 8 bor 2048) + ), [t3, a3] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, 42, a3, t3, 1), Stream = ?BACKEND:stream(State3), @@ -2752,12 +2755,12 @@ move_to_native_register_test_() -> ), Stream = ?BACKEND:stream(State2), Dump = << - " 0: 01852f83 lw t6,24(a0)\n" - " 4: 06052f03 lw t5,96(a0)\n" - " 8: 004fae83 lw t4,4(t6)\n" - " c: 01df2c23 sw t4,24(t5)\n" - " 10: 008fae83 lw t4,8(t6)\n" - " 14: 01df2e23 sw t4,28(t5)" + " 0: 01852f83 lw t6,24(a0)\n" + " 4: 06452f03 lw t5,100(a0)\n" + " 8: 004fae83 lw t4,4(t6)\n" + " c: 01df2c23 sw t4,24(t5)\n" + " 10: 008fae83 lw t4,8(t6)\n" + " 14: 01df2e23 sw t4,28(t5)" >>, ?assertStream(riscv32, Dump, Stream) end) @@ -3518,93 +3521,77 @@ add_beam_test() -> Stream = ?BACKEND:stream(State15), Dump = << - % jump table (new 8-byte format) - " 0: 00000697 auipc a3,0x0\n" - " 4: 0e068067 jr 224(a3) # 0xe0\n" - " 8: 00000697 auipc a3,0x0\n" - " c: 01868067 jr 24(a3) # 0x20\n" - " 10: 00000697 auipc a3,0x0\n" - " 14: 04868067 jr 72(a3) # 0x58\n" - " 18: 00000697 auipc a3,0x0\n" - " 1c: 0c268067 jr 194(a3) # 0xda\n" - % label 1 - % {move,{integer,9},{x,1}}. - " 20: 09f00f93 li t6,159\n" - " 24: 01f52e23 sw t6,28(a0)\n" - % {move,{integer,8},{x,0}} - " 28: 08f00f93 li t6,143\n" - " 2c: 01f52c23 sw t6,24(a0)\n" - % {call_only,2,{f,2}}. - " 30: 0085af83 lw t6,8(a1)\n" - " 34: 1ffd addi t6,t6,-1\n" - " 36: 01f5a423 sw t6,8(a1)\n" - " 3a: 000f8663 beqz t6,0x46\n" - " 3e: a829 j 0x58\n" - " 40: 0001 nop\n" - " 42: 00000013 nop\n" - " 46: 00000f97 auipc t6,0x0\n" - " 4a: 0fc9 addi t6,t6,18 # 0x58\n" - " 4c: 0001 nop\n" - " 4e: 01f5a223 sw t6,4(a1)\n" - " 52: 00862f83 lw t6,8(a2)\n" - " 56: 8f82 jr t6\n" - % label 2 - % {allocate,1,1}. - " 58: 01462f83 lw t6,20(a2)\n" - " 5c: 1141 addi sp,sp,-16\n" - " 5e: c006 sw ra,0(sp)\n" - " 60: c22a sw a0,4(sp)\n" - " 62: c42e sw a1,8(sp)\n" - " 64: c632 sw a2,12(sp)\n" - " 66: 4605 li a2,1\n" - " 68: 4681 li a3,0\n" - " 6a: 4705 li a4,1\n" - " 6c: 9f82 jalr t6\n" - " 6e: 8faa mv t6,a0\n" - " 70: 4082 lw ra,0(sp)\n" - " 72: 4512 lw a0,4(sp)\n" - " 74: 45a2 lw a1,8(sp)\n" - " 76: 4632 lw a2,12(sp)\n" - " 78: 0141 addi sp,sp,16\n" - " 7a: 01ff9f13 slli t5,t6,0x1f\n" - " 7e: 000f4763 bltz t5,0x8c\n" - " 82: 01862f83 lw t6,24(a2)\n" - " 86: 08600613 li a2,134\n" - " 8a: 8f82 jr t6\n" - % {init_yregs,{list,[{y,0}]}}. - %% move_to_vm_register(State8, ?TERM_NIL, {y_reg, 0}), - " 8c: 03b00f13 li t5,59\n" - " 90: 01452f83 lw t6,20(a0)\n" - " 94: 01efa023 sw t5,0(t6)\n" - % {call,1,{f,3}} - %% call_or_schedule_next(State9, 3), - " 98: 0005af03 lw t5,0(a1)\n" - " 9c: 000f2f03 lw t5,0(t5)\n" - " a0: 0f62 slli t5,t5,0x18\n" - " a2: 36800f93 li t6,872\n" - " a6: 00000013 nop\n" - " aa: 01ff6f33 or t5,t5,t6\n" - " ae: 05e52e23 sw t5,92(a0)\n" - " b2: 0085af83 lw t6,8(a1)\n" - " b6: 1ffd addi t6,t6,-1\n" - " b8: 01f5a423 sw t6,8(a1)\n" - " bc: 000f8663 beqz t6,0xc8\n" - " c0: a829 j 0xda\n" - " c2: 0001 nop\n" - " c4: 00000013 nop\n" - " c8: 00000f97 auipc t6,0x0\n" - " cc: 0fc9 addi t6,t6,18 # 0xda\n" - " ce: 0001 nop\n" - " d0: 01f5a223 sw t6,4(a1)\n" - " d4: 00862f83 lw t6,8(a2)\n" - " d8: 8f82 jr t6\n" - %% (continuation) - % label 3 - " da: 00462f83 lw t6,4(a2)\n" - " de: 8f82 jr t6\n" - % label 0 - " e0: 00462f83 lw t6,4(a2)\n" - " e4: 8f82 jr t6" + " 0: 00000697 auipc a3,0x0\n" + " 4: 0da68067 jr 218(a3) # 0xda\n" + " 8: 00000697 auipc a3,0x0\n" + " c: 01868067 jr 24(a3) # 0x20\n" + " 10: 00000697 auipc a3,0x0\n" + " 14: 04868067 jr 72(a3) # 0x58\n" + " 18: 00000697 auipc a3,0x0\n" + " 1c: 0bc68067 jr 188(a3) # 0xd4\n" + " 20: 09f00f93 li t6,159\n" + " 24: 01f52e23 sw t6,28(a0)\n" + " 28: 08f00f93 li t6,143\n" + " 2c: 01f52c23 sw t6,24(a0)\n" + " 30: 0085af83 lw t6,8(a1)\n" + " 34: 1ffd addi t6,t6,-1\n" + " 36: 01f5a423 sw t6,8(a1)\n" + " 3a: 000f8663 beqz t6,0x46\n" + " 3e: a829 j 0x58\n" + " 40: 0001 nop\n" + " 42: 00000013 nop\n" + " 46: 00000f97 auipc t6,0x0\n" + " 4a: 0fc9 addi t6,t6,18 # 0x58\n" + " 4c: 0001 nop\n" + " 4e: 01f5a223 sw t6,4(a1)\n" + " 52: 00862f83 lw t6,8(a2)\n" + " 56: 8f82 jr t6\n" + " 58: 01462f83 lw t6,20(a2)\n" + " 5c: 1141 addi sp,sp,-16\n" + " 5e: c006 sw ra,0(sp)\n" + " 60: c22a sw a0,4(sp)\n" + " 62: c42e sw a1,8(sp)\n" + " 64: c632 sw a2,12(sp)\n" + " 66: 4605 li a2,1\n" + " 68: 4681 li a3,0\n" + " 6a: 4705 li a4,1\n" + " 6c: 9f82 jalr t6\n" + " 6e: 8faa mv t6,a0\n" + " 70: 4082 lw ra,0(sp)\n" + " 72: 4512 lw a0,4(sp)\n" + " 74: 45a2 lw a1,8(sp)\n" + " 76: 4632 lw a2,12(sp)\n" + " 78: 0141 addi sp,sp,16\n" + " 7a: 01ff9f13 slli t5,t6,0x1f\n" + " 7e: 000f4763 bltz t5,0x8c\n" + " 82: 01862f83 lw t6,24(a2)\n" + " 86: 08600613 li a2,134\n" + " 8a: 8f82 jr t6\n" + " 8c: 03b00f13 li t5,59\n" + " 90: 01452f83 lw t6,20(a0)\n" + " 94: 01efa023 sw t5,0(t6)\n" + " 98: 0005af83 lw t6,0(a1)\n" + " 9c: 07f52023 sw t6,96(a0)\n" + " a0: 35000f93 li t6,848\n" + " a4: 00000013 nop\n" + " a8: 05f52e23 sw t6,92(a0)\n" + " ac: 0085af83 lw t6,8(a1)\n" + " b0: 1ffd addi t6,t6,-1\n" + " b2: 01f5a423 sw t6,8(a1)\n" + " b6: 000f8663 beqz t6,0xc2\n" + " ba: a829 j 0xd4\n" + " bc: 0001 nop\n" + " be: 00000013 nop\n" + " c2: 00000f97 auipc t6,0x0\n" + " c6: 0fc9 addi t6,t6,18 # 0xd4\n" + " c8: 0001 nop\n" + " ca: 01f5a223 sw t6,4(a1)\n" + " ce: 00862f83 lw t6,8(a2)\n" + " d2: 8f82 jr t6\n" + " d4: 00462f83 lw t6,4(a2)\n" + " d8: 8f82 jr t6\n" + " da: 00462f83 lw t6,4(a2)\n" + " de: 8f82 jr t6" >>, ?assertStream(riscv32, Dump, Stream). diff --git a/tests/libs/jit/jit_riscv64_tests.erl b/tests/libs/jit/jit_riscv64_tests.erl index 8e88e93217..b0d9e8d7f3 100644 --- a/tests/libs/jit/jit_riscv64_tests.erl +++ b/tests/libs/jit/jit_riscv64_tests.erl @@ -2660,9 +2660,9 @@ move_to_array_element_test_() -> A3Bit = 1 bsl 3, T3Bit = 1 bsl 11, UsedMask = A3Bit bor T3Bit, - AvailMask = element(7, State0) band (bnot UsedMask), - State1 = setelement(7, State0, AvailMask), - State2 = setelement(8, State1, UsedMask), + Regs0 = element(9, State0), + AvailMask = jit_regs:available_regs(Regs0) band (bnot UsedMask), + State2 = setelement(9, State0, jit_regs:set_masks(Regs0, AvailMask, UsedMask)), ?assertEqual(lists:sort([a3, t3]), lists:sort(?BACKEND:used_regs(State2))), State3 = ?BACKEND:move_to_array_element(State2, {x_reg, 0}, a3, t3, 1), Stream = ?BACKEND:stream(State3), @@ -2680,9 +2680,9 @@ move_to_array_element_test_() -> A3Bit = 1 bsl 3, T3Bit = 1 bsl 11, UsedMask = A3Bit bor T3Bit, - AvailMask = element(7, State0) band (bnot UsedMask), - State1 = setelement(7, State0, AvailMask), - State2 = setelement(8, State1, UsedMask), + Regs0 = element(9, State0), + AvailMask = jit_regs:available_regs(Regs0) band (bnot UsedMask), + State2 = setelement(9, State0, jit_regs:set_masks(Regs0, AvailMask, UsedMask)), ?assertEqual(lists:sort([a3, t3]), lists:sort(?BACKEND:used_regs(State2))), State3 = ?BACKEND:move_to_array_element(State2, 42, a3, t3, 1), Stream = ?BACKEND:stream(State3), diff --git a/tests/libs/jit/jit_tests_common.erl b/tests/libs/jit/jit_tests_common.erl index 21783251be..70bd75304b 100644 --- a/tests/libs/jit/jit_tests_common.erl +++ b/tests/libs/jit/jit_tests_common.erl @@ -154,6 +154,7 @@ toolchain_prefixes(xtensa) -> %% whose objdump displays raw bytes rather than instruction words, %% breaking hex_to_bin/3's little-endian conversion. ["xtensa-esp32-elf", "xtensa-esp32s2-elf", "xtensa-esp32s3-elf"] ++ + ["xtensa-lx106-elf"] ++ ["xtensa" ++ V || V <- ["-unknown-elf", "-elf", "-linux-gnu"]] ++ ["xtensa-lx6-linux-gnu"]; toolchain_prefixes(Arch) -> diff --git a/tests/libs/jit/jit_wasm32_tests.erl b/tests/libs/jit/jit_wasm32_tests.erl index 2bcc71e4ac..0698dde60a 100644 --- a/tests/libs/jit/jit_wasm32_tests.erl +++ b/tests/libs/jit/jit_wasm32_tests.erl @@ -473,22 +473,22 @@ set_bs_test() -> State5 = ?BACKEND:return_labels_and_lines(State4, []), Stream = ?BACKEND:stream(State5), Dump = << - " 0000e0: 08 7f local[3..10] type=i32\n" - " 0000e2: 20 00 local.get 0\n" - " 0000e4: 28 02 18 i32.load 2 24\n" - " 0000e7: 21 03 local.set 3\n" - " 0000e9: 20 00 local.get 0\n" - " 0000eb: 20 03 local.get 3\n" - " 0000ed: 36 02 64 i32.store 2 100\n" - " 0000f0: 20 00 local.get 0\n" - " 0000f2: 41 00 i32.const 0\n" - " 0000f4: 36 02 68 i32.store 2 104\n" - " 0000f7: 20 00 local.get 0\n" - " 0000f9: 0f return\n" - " 0000fa: 0b end\n" - " 0000fd: 08 7f local[3..10] type=i32\n" - " 0000ff: 20 00 local.get 0\n" - " 000101: 0b end" + " 0000b3: 08 7f local[3..10] type=i32\n" + " 0000b5: 20 00 local.get 0\n" + " 0000b7: 28 02 18 i32.load 2 24\n" + " 0000ba: 21 03 local.set 3\n" + " 0000bc: 20 00 local.get 0\n" + " 0000be: 20 03 local.get 3\n" + " 0000c0: 36 02 68 i32.store 2 104\n" + " 0000c3: 20 00 local.get 0\n" + " 0000c5: 41 00 i32.const 0\n" + " 0000c7: 36 02 6c i32.store 2 108\n" + " 0000ca: 20 00 local.get 0\n" + " 0000cc: 0f return\n" + " 0000cd: 0b end\n" + " 0000d0: 08 7f local[3..10] type=i32\n" + " 0000d2: 20 00 local.get 0\n" + " 0000d4: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -1409,66 +1409,58 @@ call_or_schedule_next_test() -> State4 = ?BACKEND:return_labels_and_lines(State3, []), Stream = ?BACKEND:stream(State4), Dump = << - " 0000f2: 08 7f local[3..10] type=i32\n" - " 0000f4: 20 01 local.get 1\n" - " 0000f6: 28 02 00 i32.load 2 0\n" - " 0000f9: 28 02 00 i32.load 2 0\n" - " 0000fc: 21 03 local.set 3\n" - " 0000fe: 20 03 local.get 3\n" - " 000100: 41 18 i32.const 24\n" - " 000102: 74 i32.shl\n" - " 000103: 21 03 local.set 3\n" - " 000105: 41 c0 00 i32.const 64\n" - " 000108: 21 04 local.set 4\n" - " 00010a: 20 03 local.get 3\n" - " 00010c: 20 04 local.get 4\n" - " 00010e: 72 i32.or\n" - " 00010f: 21 03 local.set 3\n" - " 000111: 20 00 local.get 0\n" - " 000113: 20 03 local.get 3\n" - " 000115: 36 02 5c i32.store 2 92\n" - " 000118: 20 01 local.get 1\n" - " 00011a: 28 02 08 i32.load 2 8\n" - " 00011d: 41 01 i32.const 1\n" - " 00011f: 6b i32.sub\n" - " 000120: 21 03 local.set 3\n" - " 000122: 20 01 local.get 1\n" - " 000124: 20 03 local.get 3\n" - " 000126: 36 02 08 i32.store 2 8\n" - " 000129: 20 03 local.get 3\n" - " 00012b: 45 i32.eqz\n" - " 00012c: 04 40 if\n" - " 00012e: 20 01 local.get 1\n" - " 000130: 41 02 i32.const 2\n" - " 000132: 36 02 04 i32.store 2 4\n" - " 000135: 20 00 local.get 0\n" - " 000137: 20 01 local.get 1\n" - " 000139: 20 02 local.get 2\n" - " 00013b: 28 02 08 i32.load 2 8\n" - " 00013e: 11 01 00 call_indirect 0 (type 1)\n" - " 000141: 0f return\n" - " 000142: 0b end\n" - " 000143: 20 01 local.get 1\n" - " 000145: 41 02 i32.const 2\n" - " 000147: 36 02 04 i32.store 2 4\n" - " 00014a: 20 00 local.get 0\n" - " 00014c: 0f return\n" - " 00014d: 20 00 local.get 0\n" - " 00014f: 0f return\n" - " 000150: 0b end\n" - " 000153: 08 7f local[3..10] type=i32\n" - " 000155: 20 00 local.get 0\n" - " 000157: 0b end\n" - " 00015a: 08 7f local[3..10] type=i32\n" - " 00015c: 20 00 local.get 0\n" - " 00015e: 0b end\n" - " 000161: 08 7f local[3..10] type=i32\n" - " 000163: 20 00 local.get 0\n" - " 000165: 0b end\n" - " 000168: 08 7f local[3..10] type=i32\n" - " 00016a: 20 00 local.get 0\n" - " 00016c: 0f return\n" - " 00016d: 0b end" + " 0000c5: 08 7f local[3..10] type=i32\n" + " 0000c7: 20 01 local.get 1\n" + " 0000c9: 28 02 00 i32.load 2 0\n" + " 0000cc: 21 03 local.set 3\n" + " 0000ce: 20 00 local.get 0\n" + " 0000d0: 41 c0 00 i32.const 64\n" + " 0000d3: 36 02 5c i32.store 2 92\n" + " 0000d6: 20 00 local.get 0\n" + " 0000d8: 20 03 local.get 3\n" + " 0000da: 36 02 60 i32.store 2 96\n" + " 0000dd: 20 01 local.get 1\n" + " 0000df: 28 02 08 i32.load 2 8\n" + " 0000e2: 41 01 i32.const 1\n" + " 0000e4: 6b i32.sub\n" + " 0000e5: 21 03 local.set 3\n" + " 0000e7: 20 01 local.get 1\n" + " 0000e9: 20 03 local.get 3\n" + " 0000eb: 36 02 08 i32.store 2 8\n" + " 0000ee: 20 03 local.get 3\n" + " 0000f0: 45 i32.eqz\n" + " 0000f1: 04 40 if\n" + " 0000f3: 20 01 local.get 1\n" + " 0000f5: 41 02 i32.const 2\n" + " 0000f7: 36 02 04 i32.store 2 4\n" + " 0000fa: 20 00 local.get 0\n" + " 0000fc: 20 01 local.get 1\n" + " 0000fe: 20 02 local.get 2\n" + " 000100: 28 02 08 i32.load 2 8\n" + " 000103: 11 01 00 call_indirect 0 (type 1)\n" + " 000106: 0f return\n" + " 000107: 0b end\n" + " 000108: 20 01 local.get 1\n" + " 00010a: 41 02 i32.const 2\n" + " 00010c: 36 02 04 i32.store 2 4\n" + " 00010f: 20 00 local.get 0\n" + " 000111: 0f return\n" + " 000112: 20 00 local.get 0\n" + " 000114: 0f return\n" + " 000115: 0b end\n" + " 000118: 08 7f local[3..10] type=i32\n" + " 00011a: 20 00 local.get 0\n" + " 00011c: 0b end\n" + " 00011f: 08 7f local[3..10] type=i32\n" + " 000121: 20 00 local.get 0\n" + " 000123: 0b end\n" + " 000126: 08 7f local[3..10] type=i32\n" + " 000128: 20 00 local.get 0\n" + " 00012a: 0b end\n" + " 00012d: 08 7f local[3..10] type=i32\n" + " 00012f: 20 00 local.get 0\n" + " 000131: 0f return\n" + " 000132: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -1534,22 +1526,28 @@ move_to_cp_test() -> State4 = ?BACKEND:return_labels_and_lines(State3, []), Stream = ?BACKEND:stream(State4), Dump = << - " 0000e0: 08 7f local[3..10] type=i32\n" - " 0000e2: 20 00 local.get 0\n" - " 0000e4: 28 02 14 i32.load 2 20\n" - " 0000e7: 21 03 local.set 3\n" - " 0000e9: 20 03 local.get 3\n" - " 0000eb: 28 02 00 i32.load 2 0\n" - " 0000ee: 21 03 local.set 3\n" - " 0000f0: 20 00 local.get 0\n" - " 0000f2: 20 03 local.get 3\n" - " 0000f4: 36 02 5c i32.store 2 92\n" - " 0000f7: 20 00 local.get 0\n" - " 0000f9: 0f return\n" - " 0000fa: 0b end\n" - " 0000fd: 08 7f local[3..10] type=i32\n" - " 0000ff: 20 00 local.get 0\n" - " 000101: 0b end" + " 0000b3: 08 7f local[3..10] type=i32\n" + " 0000b5: 20 00 local.get 0\n" + " 0000b7: 28 02 14 i32.load 2 20\n" + " 0000ba: 21 03 local.set 3\n" + " 0000bc: 20 03 local.get 3\n" + " 0000be: 28 02 00 i32.load 2 0\n" + " 0000c1: 21 04 local.set 4\n" + " 0000c3: 20 00 local.get 0\n" + " 0000c5: 20 04 local.get 4\n" + " 0000c7: 36 02 5c i32.store 2 92\n" + " 0000ca: 20 03 local.get 3\n" + " 0000cc: 28 02 04 i32.load 2 4\n" + " 0000cf: 21 04 local.set 4\n" + " 0000d1: 20 00 local.get 0\n" + " 0000d3: 20 04 local.get 4\n" + " 0000d5: 36 02 60 i32.store 2 96\n" + " 0000d8: 20 00 local.get 0\n" + " 0000da: 0f return\n" + " 0000db: 0b end\n" + " 0000de: 08 7f local[3..10] type=i32\n" + " 0000e0: 20 00 local.get 0\n" + " 0000e2: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -2011,40 +2009,32 @@ call_primitive_with_cp_test() -> State4 = ?BACKEND:return_labels_and_lines(State3, []), Stream = ?BACKEND:stream(State4), Dump = << - " 0000e6: 08 7f local[3..10] type=i32\n" - " 0000e8: 20 01 local.get 1\n" - " 0000ea: 28 02 00 i32.load 2 0\n" - " 0000ed: 28 02 00 i32.load 2 0\n" - " 0000f0: 21 03 local.set 3\n" - " 0000f2: 20 03 local.get 3\n" - " 0000f4: 41 18 i32.const 24\n" - " 0000f6: 74 i32.shl\n" - " 0000f7: 21 03 local.set 3\n" - " 0000f9: 41 20 i32.const 32\n" - " 0000fb: 21 04 local.set 4\n" - " 0000fd: 20 03 local.get 3\n" - " 0000ff: 20 04 local.get 4\n" - " 000101: 72 i32.or\n" - " 000102: 21 03 local.set 3\n" - " 000104: 20 00 local.get 0\n" - " 000106: 20 03 local.get 3\n" - " 000108: 36 02 5c i32.store 2 92\n" - " 00010b: 20 00 local.get 0\n" - " 00010d: 20 01 local.get 1\n" - " 00010f: 20 02 local.get 2\n" - " 000111: 28 02 00 i32.load 2 0\n" - " 000114: 11 01 00 call_indirect 0 (type 1)\n" - " 000117: 0f return\n" - " 000118: 20 00 local.get 0\n" - " 00011a: 0f return\n" - " 00011b: 0b end\n" - " 00011e: 08 7f local[3..10] type=i32\n" - " 000120: 20 00 local.get 0\n" - " 000122: 0b end\n" - " 000125: 08 7f local[3..10] type=i32\n" - " 000127: 20 00 local.get 0\n" - " 000129: 0f return\n" - " 00012a: 0b end" + " 0000b9: 08 7f local[3..10] type=i32\n" + " 0000bb: 20 01 local.get 1\n" + " 0000bd: 28 02 00 i32.load 2 0\n" + " 0000c0: 21 03 local.set 3\n" + " 0000c2: 20 00 local.get 0\n" + " 0000c4: 41 20 i32.const 32\n" + " 0000c6: 36 02 5c i32.store 2 92\n" + " 0000c9: 20 00 local.get 0\n" + " 0000cb: 20 03 local.get 3\n" + " 0000cd: 36 02 60 i32.store 2 96\n" + " 0000d0: 20 00 local.get 0\n" + " 0000d2: 20 01 local.get 1\n" + " 0000d4: 20 02 local.get 2\n" + " 0000d6: 28 02 00 i32.load 2 0\n" + " 0000d9: 11 01 00 call_indirect 0 (type 1)\n" + " 0000dc: 0f return\n" + " 0000dd: 20 00 local.get 0\n" + " 0000df: 0f return\n" + " 0000e0: 0b end\n" + " 0000e3: 08 7f local[3..10] type=i32\n" + " 0000e5: 20 00 local.get 0\n" + " 0000e7: 0b end\n" + " 0000ea: 08 7f local[3..10] type=i32\n" + " 0000ec: 20 00 local.get 0\n" + " 0000ee: 0f return\n" + " 0000ef: 0b end" >>, ?assertStream(wasm32, Dump, Stream). @@ -2242,6 +2232,17 @@ condition_int_cast_ne_test() -> >>, ?assertStream(wasm32, Dump, Stream). +call_func_ptr_regs_test() -> + State0 = ?BACKEND:new(0, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), + State2 = ?BACKEND:add_label(State1, 0), + {State3, FuncPtr} = ?BACKEND:move_to_native_register(State2, {x_reg, 0}), + ?assertEqual([FuncPtr], ?BACKEND:used_regs(State3)), + {State4, ResultReg} = ?BACKEND:call_func_ptr(State3, {free, FuncPtr}, [ctx]), + ?assertEqual([ResultReg], ?BACKEND:used_regs(State4)), + State5 = ?BACKEND:free_native_registers(State4, [ResultReg]), + ?assertEqual(ok, ?BACKEND:assert_all_native_free(State5)). + call_func_ptr_primitive_test() -> State0 = ?BACKEND:new(0, jit_stream_binary, jit_stream_binary:new(0)), State1 = ?BACKEND:jump_table(State0, 1), @@ -2456,3 +2457,27 @@ alloc_local_overflow_reuse_test() -> " 00017f: 0b end" >>, ?assertStream(wasm32, Dump, Stream). + +if_block_pool_growth_merge_test() -> + State0 = ?BACKEND:new(0, jit_stream_binary, jit_stream_binary:new(65536)), + State1 = ?BACKEND:jump_table(State0, 2), + State2 = ?BACKEND:add_label(State1, 1), + {State3, Cond} = ?BACKEND:move_to_native_register(State2, {x_reg, 0}), + %% The body allocates 9 locals (growing the pool from 8 to 9) and frees them + %% all, so every scratch local must be free again after the merge. + State4 = ?BACKEND:if_block(State3, {{free, Cond}, '==', {free, Cond}}, fun(BSt0) -> + {BSt1, Regs} = lists:foldl( + fun(_, {S, Acc}) -> + {S1, R} = ?BACKEND:call_primitive(S, 0, [ctx]), + {S1, [R | Acc]} + end, + {BSt0, []}, + lists:seq(1, 9) + ), + ?BACKEND:free_native_registers(BSt1, Regs) + end), + ?assertEqual([], ?BACKEND:used_regs(State4)), + Available = ?BACKEND:available_regs(State4), + ?assertEqual(9, length(Available)), + ?assert(lists:member(local11, Available)), + ?assertEqual(ok, ?BACKEND:assert_all_native_free(State4)). diff --git a/tests/libs/jit/jit_xtensa_tests.erl b/tests/libs/jit/jit_xtensa_tests.erl index 2a7b677f16..8b5198db14 100644 --- a/tests/libs/jit/jit_xtensa_tests.erl +++ b/tests/libs/jit/jit_xtensa_tests.erl @@ -396,7 +396,10 @@ move_to_cp_test() -> Dump = << " 0: 0522e2 l32i a14, a2, 20\n" " 3: 002ef2 l32i a15, a14, 0\n" - " 6: 1762f2 s32i a15, a2, 92" + " 6: 1762f2 s32i a15, a2, 92\n" + " 9: 0522e2 l32i a14, a2, 20\n" + " c: 012ef2 l32i a15, a14, 4\n" + " f: 1862f2 s32i a15, a2, 96" >>, ?assertStream(xtensa, Dump, Stream). diff --git a/tests/libs/jit/tests.erl b/tests/libs/jit/tests.erl index 7c5c53d43a..ae7c6873db 100644 --- a/tests/libs/jit/tests.erl +++ b/tests/libs/jit/tests.erl @@ -28,6 +28,7 @@ start() -> "BEAM" = erlang:system_info(machine), Result = etest:test([ jit_tests, + jit_regs_tests, jit_dwarf_tests, jit_aarch64_tests, jit_aarch64_asm_tests, diff --git a/tests/test-structs.c b/tests/test-structs.c index 52ce22dcd2..b4b730a4a3 100644 --- a/tests/test-structs.c +++ b/tests/test-structs.c @@ -22,6 +22,8 @@ #include #include "atom_table.h" +#include "module.h" +#include "term.h" #include "utils.h" #include "valueshashtable.h" @@ -479,6 +481,48 @@ void test_atom_table(void) atom_table_destroy(table); } +static void test_cp_encoding(void) +{ + // A stack Module is suitably aligned; only module_index is read by make_cp. + Module m; + memset(&m, 0, sizeof(m)); + m.module_index = 300; // >= 256: the case the old 8-bit packing overflowed. + + // Offsets up to the 64-bit packing limit (offset << 2 must fit in 24 bits). + unsigned int offsets[] = { 0, 4, 1000, (1u << 20), (1u << 22) - 4 }; + for (size_t i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { + unsigned int off = offsets[i]; + cp_t cp = make_cp(&m, off); + + // Offset round-trips. + assert(cp_to_offset(cp) == off); + + // Module identity round-trips. +#if TERM_BITS == 64 + // 64-bit: the module index is packed in the high bits. + assert((unsigned int) (cp >> 24) == 300u); +#else + // 32-bit: the Module pointer is stored directly (no index lookup). + assert(cp_to_module(cp, NULL) == &m); +#endif + + // store_cp/load_cp round-trip across the on-stack representation. + term slots[2] = { 0, 0 }; + store_cp(slots, cp); + assert(load_cp(slots) == cp); + + // Every stored slot must be GC-safe (low 2 bits clear => TERM_PRIMARY_CP), + // so the collector skips it instead of following it as a pointer. + for (int s = 0; s < CP_SIZE_IN_TERMS; s++) { + assert((slots[s] & TERM_PRIMARY_MASK) == TERM_PRIMARY_CP); + } + } + + // The process-termination sentinel is recognized; a real cp is not. + assert(cp_is_terminate((cp_t) -1)); + assert(!cp_is_terminate(make_cp(&m, 0))); +} + int main(int argc, char **argv) { UNUSED(argc); @@ -486,6 +530,7 @@ int main(int argc, char **argv) test_valueshashtable(); test_atom_table(); + test_cp_encoding(); return EXIT_SUCCESS; } diff --git a/tests/test.c b/tests/test.c index 342c142fd4..d4ed1bf4f0 100644 --- a/tests/test.c +++ b/tests/test.c @@ -562,6 +562,7 @@ struct Test tests[] = { TEST_CASE(test_code_all_available_loaded), TEST_CASE_EXPECTED(test_code_load_binary, 24), + TEST_CASE(test_many_modules), TEST_CASE_EXPECTED(test_code_load_abs, 24), TEST_CASE(test_code_ensure_loaded), TEST_CASE_ATOMVM_ONLY(test_add_avm_pack_binary, 24),