diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/MergeExplodeKeyTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/MergeExplodeKeyTest.java new file mode 100644 index 000000000000..f55879f8a152 --- /dev/null +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/MergeExplodeKeyTest.java @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.truffle.test; + +import org.junit.Test; + +import com.oracle.truffle.api.CompilerAsserts; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; +import com.oracle.truffle.api.frame.FrameDescriptor; +import com.oracle.truffle.api.frame.FrameSlotKind; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.ExplodeLoop; +import com.oracle.truffle.api.nodes.ExplodeLoop.LoopExplosionKind; +import com.oracle.truffle.api.nodes.RootNode; + +import jdk.vm.ci.code.BailoutException; + +/** + * Tests the `CompilerDirectives.mergeExplodeKey` system for explicit merge explode keys. + */ +@SuppressWarnings("deprecation") +public class MergeExplodeKeyTest extends PartialEvaluationTest { + + public static class Bytecode { + public static final byte CONST = 0; + public static final byte ARGUMENT = 1; + public static final byte ADD = 2; + public static final byte SUB = 3; + public static final byte DUP = 4; + public static final byte POP = 5; + public static final byte JMP = 6; + public static final byte IFZERO = 7; + public static final byte RETURN = 8; + } + + public static class Program extends RootNode { + protected final String name; + @CompilationFinal(dimensions = 1) protected final byte[] bytecodes; + protected final int stackOffset; + protected final boolean markTopAsKey; + + static Program create(String name, byte[] bytecodes, int maxStack, boolean markTopAsKey) { + var builder = FrameDescriptor.newBuilder(); + int stackOffset = builder.addSlots(maxStack, FrameSlotKind.Int); + return new Program(name, bytecodes, builder.build(), stackOffset, markTopAsKey); + } + + Program(String name, byte[] bytecodes, FrameDescriptor descriptor, int stackOffset, boolean markTopAsKey) { + super(null, descriptor); + this.name = name; + this.bytecodes = bytecodes; + this.stackOffset = stackOffset; + this.markTopAsKey = markTopAsKey; + } + + protected void setInt(VirtualFrame frame, int stackIndex, int value) { + frame.setInt(stackOffset + stackIndex, value); + } + + protected int getInt(VirtualFrame frame, int stackIndex) { + return frame.getInt(stackOffset + stackIndex); + } + + @Override + public String toString() { + return name; + } + + @Override + @ExplodeLoop(kind = LoopExplosionKind.MERGE_EXPLODE) + public Object execute(VirtualFrame frame) { + int top = -1; + int bci = 0; + bci = CompilerDirectives.mergeExplodeKey(bci); + if (markTopAsKey) { + /* Testing error on multiple key variables. */ + top = CompilerDirectives.mergeExplodeKey(top); + } + while (true) { + CompilerAsserts.partialEvaluationConstant(bci); + switch (bytecodes[bci]) { + case Bytecode.CONST: { + byte value = bytecodes[bci + 1]; + top++; + setInt(frame, top, value); + bci = bci + 2; + continue; + } + case Bytecode.ARGUMENT: { + int value = (int) frame.getArguments()[bytecodes[bci + 1]]; + top++; + setInt(frame, top, value); + bci = bci + 2; + continue; + } + + case Bytecode.ADD: { + int left = getInt(frame, top); + int right = getInt(frame, top - 1); + top--; + setInt(frame, top, left + right); + bci = bci + 1; + continue; + } + case Bytecode.SUB: { + int left = getInt(frame, top); + int right = getInt(frame, top - 1); + top--; + setInt(frame, top, left - right); + bci = bci + 1; + continue; + } + case Bytecode.DUP: { + int dupValue = getInt(frame, top); + top++; + setInt(frame, top, dupValue); + bci++; + continue; + } + case Bytecode.POP: { + top--; + bci++; + continue; + } + + case Bytecode.JMP: { + byte newBci = bytecodes[bci + 1]; + bci = newBci; + continue; + } + case Bytecode.IFZERO: { + int value = getInt(frame, top); + top--; + if (value == 0) { + bci = bytecodes[bci + 1]; + continue; + } else { + bci = bci + 2; + continue; + } + } + case Bytecode.RETURN: { + int value = getInt(frame, top); + return value; + } + + default: { + throw new IllegalStateException(); + } + } + } + } + } + + @Test + public void constReturnProgram() { + byte[] bytecodes = new byte[]{ + /* 0: */Bytecode.CONST, + /* 1: */42, + /* 2: */Bytecode.RETURN}; + partialEval(Program.create("constReturnProgram", bytecodes, 2, false)); + } + + @Test + public void constAddProgram() { + byte[] bytecodes = new byte[]{ + /* 0: */Bytecode.CONST, + /* 1: */40, + /* 2: */Bytecode.CONST, + /* 3: */2, + /* 4: */Bytecode.ADD, + /* 5: */Bytecode.RETURN}; + partialEval(Program.create("constAddProgram", bytecodes, 2, false)); + } + + @Test(expected = BailoutException.class) + public void multipleKeyVariables() { + byte[] bytecodes = new byte[]{ + /* 0: */Bytecode.CONST, + /* 1: */42, + /* 2: */Bytecode.RETURN}; + partialEval(Program.create("multipleKeyVariables", bytecodes, 2, true)); + } + + @Test(expected = BailoutException.class) + public void variableStackSize() { + byte[] bytecodes = new byte[]{ + /* 0: */Bytecode.ARGUMENT, + /* 1: */0, + /* 2: */Bytecode.IFZERO, + /* 3: */6, + /* 4: */Bytecode.CONST, + /* 5: */40, + /* 6: */Bytecode.CONST, + /* 7: */42, + /* 8: */Bytecode.RETURN}; + partialEval(Program.create("variableStackSize", bytecodes, 3, false), 0); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphDecoder.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphDecoder.java index 6591a29eba7c..07a13a86108f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphDecoder.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphDecoder.java @@ -32,10 +32,12 @@ import java.util.BitSet; import java.util.Deque; import java.util.EnumSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import java.util.stream.Collectors; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; @@ -85,8 +87,9 @@ public class GraphDecoder { /** Decoding state maintained for each encoded graph. */ protected class MethodScope { - /** The loop that contains the call. Only non-null during method inlining. */ - public final LoopScope callerLoopScope; + /** The method that contains the call. Only non-null during method inlining. */ + public final MethodScope caller; + /** * Mark for nodes that were present before the decoding of this method started. Note that * nodes that were decoded after the mark can still be part of an outer method, since @@ -107,12 +110,21 @@ protected class MethodScope { /** The kind of loop explosion to be performed during decoding. */ public final LoopExplosionPlugin.LoopExplosionKind loopExplosion; + public LoopScope currentLoopScope; + /** All return nodes encountered during decoding. */ public final List returnAndUnwindNodes; /** All merges created during loop explosion. */ public final EconomicSet loopExplosionMerges; + /** + * Information about already processed loop iterations for state merging during loop + * explosion. Only used when {@link MethodScope#loopExplosion} is + * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#MERGE_EXPLODE}. + */ + public final EconomicMap iterationStates; + /** * The start of explosion, and the merge point for when irreducible loops are detected. Only * used when {@link MethodScope#loopExplosion} is @@ -142,8 +154,8 @@ protected class MethodScope { public InliningLogCodec.InliningLogDecoder inliningLogDecoder; @SuppressWarnings("unchecked") - protected MethodScope(LoopScope callerLoopScope, StructuredGraph graph, EncodedGraph encodedGraph, LoopExplosionPlugin.LoopExplosionKind loopExplosion) { - this.callerLoopScope = callerLoopScope; + protected MethodScope(MethodScope caller, StructuredGraph graph, EncodedGraph encodedGraph, LoopExplosionPlugin.LoopExplosionKind loopExplosion) { + this.caller = caller; this.methodStartMark = graph.getMark(); this.encodedGraph = encodedGraph; this.loopExplosion = loopExplosion; @@ -154,7 +166,7 @@ protected MethodScope(LoopScope callerLoopScope, StructuredGraph graph, EncodedG maxFixedNodeOrderId = reader.getUVInt(); GraphState.GuardsStage guardsStage = (GraphState.GuardsStage) readObject(this); EnumSet stageFlags = (EnumSet) readObject(this); - if (callerLoopScope == null) { + if (caller == null) { /** * Only propagate stage flags in non-inlining scenarios. If the caller scope has * not been guard lowered yet (or is a runtime compilation) while we inline @@ -194,15 +206,12 @@ protected MethodScope(LoopScope callerLoopScope, StructuredGraph graph, EncodedG orderIdWidth = 0; } - if (loopExplosion.useExplosion()) { - loopExplosionMerges = EconomicSet.create(Equivalence.IDENTITY); - } else { - loopExplosionMerges = null; - } + loopExplosionMerges = loopExplosion.useExplosion() ? EconomicSet.create(Equivalence.IDENTITY) : null; + iterationStates = loopExplosion.mergeLoops() ? EconomicMap.create(Equivalence.DEFAULT) : null; } - public boolean isInlinedMethod() { - return false; + public final boolean isInlinedMethod() { + return caller != null; } public NodeSourcePosition getCallerNodeSourcePosition() { @@ -212,19 +221,6 @@ public NodeSourcePosition getCallerNodeSourcePosition() { public NodeSourcePosition getNodeSourcePosition(NodeSourcePosition position) { return position; } - - /** - * Sets the {@link #inliningLog} and {@link #optimizationLog} as the logs of the - * {@link #graph} if they are non-null. - */ - public void replaceLogsForDecodedGraph() { - if (inliningLog != null) { - graph.setInliningLog(inliningLog); - } - if (optimizationLog != null) { - graph.setOptimizationLog(optimizationLog); - } - } } /** @@ -267,7 +263,7 @@ public enum LoopScopeTrigger { } /** Decoding state maintained for each loop in the encoded graph. */ - protected static class LoopScope { + protected static final class LoopScope { public final MethodScope methodScope; public final LoopScope outer; public final int loopDepth; @@ -276,34 +272,14 @@ protected static class LoopScope { /** * Creation trigger of this particular loop scope, i.e., the reason it was created. */ - final LoopScopeTrigger trigger; + public final LoopScopeTrigger trigger; /** - * Upcoming, not yet processed, loop iterations created in the context of code duplication - * along loop exits. Only used when {@link MethodScope#loopExplosion} has - * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#duplicateLoopExits()} - * enabled. + * Upcoming, not yet processed, loop iterations. */ - public Deque nextIterationFromLoopExitDuplication; + public final LoopIterationQueue nextIterations; /** - * Same as {@link #nextIterationFromLoopExitDuplication} except that upcoming iterations - * have been created because the duplication of loop ends - * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#duplicateLoopEnds()} - * is enabled. - */ - public Deque nextIterationFromLoopEndDuplication; - /** - * Same as {@link #nextIterationFromLoopExitDuplication} except that upcoming iterations - * have been created because the unrolling of a loop with constant iteration count - * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#unrollLoops()} - * is enabled. + * Node order ID of the LoopBegin that created this loop scope. */ - public Deque nextIterationsFromUnrolling; - /** - * Information about already processed loop iterations for state merging during loop - * explosion. Only used when {@link MethodScope#loopExplosion} is - * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#MERGE_EXPLODE}. - */ - public final EconomicMap iterationStates; public final int loopBeginOrderId; /** * The worklist of fixed nodes to process. Since we already the correct processing order @@ -330,22 +306,21 @@ protected static class LoopScope { */ public final BitSet writtenNodes; + public List loopExplosionMergeKeySlots; + protected LoopScope(MethodScope methodScope) { this.methodScope = methodScope; this.outer = null; - this.nextIterationFromLoopExitDuplication = methodScope.loopExplosion.duplicateLoopExits() || methodScope.loopExplosion.mergeLoops() ? new ArrayDeque<>(2) : null; - this.nextIterationFromLoopEndDuplication = methodScope.loopExplosion.duplicateLoopEnds() ? new ArrayDeque<>(2) : null; - this.nextIterationsFromUnrolling = methodScope.loopExplosion.unrollLoops() ? new ArrayDeque<>(2) : null; this.loopDepth = 0; this.loopIteration = 0; - this.iterationStates = null; + this.trigger = LoopScopeTrigger.START; + this.nextIterations = new LoopIterationQueue(methodScope.loopExplosion); this.loopBeginOrderId = -1; - int nodeCount = methodScope.encodedGraph.nodeStartOffsets.length; this.nodesToProcess = new BitSet(methodScope.maxFixedNodeOrderId); - this.createdNodes = new Node[nodeCount]; + this.createdNodes = new Node[methodScope.encodedGraph.nodeStartOffsets.length]; this.initialCreatedNodes = null; - this.trigger = LoopScopeTrigger.START; this.writtenNodes = null; + this.loopExplosionMergeKeySlots = null; } /** @@ -356,11 +331,9 @@ protected LoopScope(MethodScope methodScope) { * created nodes instead of the initial nodes. */ protected LoopScope(MethodScope methodScope, LoopScope outer, int loopDepth, int loopIteration, int loopBeginOrderId, LoopScopeTrigger trigger, Node[] initialCreatedNodes, Node[] createdNodes, - Deque nextIterationFromLoopExitDuplication, - Deque nextIterationFromLoopEndDuplication, - Deque nextIterationsFromUnrolling, EconomicMap iterationStates) { - this(methodScope, outer, loopDepth, loopIteration, loopBeginOrderId, trigger, initialCreatedNodes, createdNodes, nextIterationFromLoopExitDuplication, nextIterationFromLoopEndDuplication, - nextIterationsFromUnrolling, iterationStates, true); + LoopIterationQueue nextIterations, + List loopExplosionMergeKeySlots) { + this(methodScope, outer, loopDepth, loopIteration, loopBeginOrderId, trigger, initialCreatedNodes, createdNodes, nextIterations, loopExplosionMergeKeySlots, true); } /** @@ -373,19 +346,15 @@ protected LoopScope(MethodScope methodScope, LoopScope outer, int loopDepth, int * {@code false}, all read accesses use the created nodes. */ protected LoopScope(MethodScope methodScope, LoopScope outer, int loopDepth, int loopIteration, int loopBeginOrderId, LoopScopeTrigger trigger, Node[] initialCreatedNodes, Node[] createdNodes, - Deque nextIterationFromLoopExitDuplication, - Deque nextIterationFromLoopEndDuplication, - Deque nextIterationsFromUnrolling, EconomicMap iterationStates, + LoopIterationQueue nextIterations, + List loopExplosionMergeKeySlots, boolean reuseInitialNodes) { this.methodScope = methodScope; this.outer = outer; this.loopDepth = loopDepth; this.loopIteration = loopIteration; this.trigger = trigger; - this.nextIterationFromLoopExitDuplication = nextIterationFromLoopExitDuplication; - this.nextIterationFromLoopEndDuplication = nextIterationFromLoopEndDuplication; - this.nextIterationsFromUnrolling = nextIterationsFromUnrolling; - this.iterationStates = iterationStates; + this.nextIterations = nextIterations; this.loopBeginOrderId = loopBeginOrderId; this.nodesToProcess = new BitSet(methodScope.maxFixedNodeOrderId); this.initialCreatedNodes = initialCreatedNodes; @@ -395,21 +364,7 @@ protected LoopScope(MethodScope methodScope, LoopScope outer, int loopDepth, int } else { this.writtenNodes = null; } - } - - /** - * Sets a node in the initial nodes of this scope and sets the written status of the node - * orderId to {@code false}. Reads of the node orderId after calling this method will access - * the initial nodes instead of the created nodes. - * - * @param nodeOrderId The orderId of the node - * @param node The node to be set in the initial nodes - */ - public void clearNode(int nodeOrderId, Node node) { - if (writtenNodes != null) { - writtenNodes.clear(nodeOrderId); - } - initialCreatedNodes[nodeOrderId] = node; + this.loopExplosionMergeKeySlots = loopExplosionMergeKeySlots; } /** @@ -488,53 +443,72 @@ public Node[] materializeCreatedNodes() { public String toString() { return loopDepth + "," + loopIteration + (loopBeginOrderId == -1 ? "" : "#" + loopBeginOrderId) + " triggered by " + trigger; } + } + protected static final class LoopIterationQueue { /** - * Determines if iterations generated when decoding this loop have yet to be processed. - * - * @return {@code true} if there are iterations to be decoded, {@code false} else + * Upcoming, not yet processed, loop iterations created in the context of code duplication + * along loop exits. Only used when {@link MethodScope#loopExplosion} has + * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#duplicateLoopExits()} + * enabled. + */ + public final Deque fromLoopExitDuplication; + + /** + * Same as {@link #fromLoopExitDuplication} except that upcoming iterations have been + * created because the duplication of loop ends + * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#duplicateLoopEnds()} + * is enabled. + */ + public final Deque fromLoopEndDuplication; + + /** + * Same as {@link #fromLoopExitDuplication} except that upcoming iterations have been + * created because the unrolling of a loop with constant iteration count + * {@link jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind#unrollLoops()} + * is enabled. */ - public boolean hasIterationsToProcess() { - return nextIterationFromLoopEndDuplication != null && !nextIterationFromLoopEndDuplication.isEmpty() || - nextIterationFromLoopExitDuplication != null && !nextIterationFromLoopExitDuplication.isEmpty() || - nextIterationsFromUnrolling != null && !nextIterationsFromUnrolling.isEmpty(); + public final Deque fromUnrolling; + + public LoopIterationQueue(LoopExplosionPlugin.LoopExplosionKind loopExplosion) { + this.fromLoopExitDuplication = loopExplosion.duplicateLoopExits() || loopExplosion.mergeLoops() ? new ArrayDeque<>(2) : null; + this.fromLoopEndDuplication = loopExplosion.duplicateLoopEnds() ? new ArrayDeque<>(2) : null; + this.fromUnrolling = loopExplosion.unrollLoops() ? new ArrayDeque<>(2) : null; } /** * Return the next iteration yet to be processed that has been created in the context of * decoding this loop scope. * - * @param remove determines if the query of the next iteration should remove it from the - * list of iterations to be processed * @return the next {@link LoopScope} to be processed that has been created in the context - * of decoding this loop scope. Note that the order is not necessarily reflecting - * the number of loop iterations. + * of decoding this loop scope, or null if there is none. Note that the order is not + * necessarily reflecting the number of loop iterations. */ - public LoopScope getNextIterationToProcess(boolean remove) { - if (nextIterationFromLoopEndDuplication != null && !nextIterationFromLoopEndDuplication.isEmpty()) { - return remove ? nextIterationFromLoopEndDuplication.removeFirst() : nextIterationFromLoopEndDuplication.peekFirst(); + public LoopScope takeNext() { + if (fromLoopEndDuplication != null && !fromLoopEndDuplication.isEmpty()) { + return fromLoopEndDuplication.removeFirst(); } - if (nextIterationFromLoopExitDuplication != null && !nextIterationFromLoopExitDuplication.isEmpty()) { - return remove ? nextIterationFromLoopExitDuplication.removeFirst() : nextIterationFromLoopExitDuplication.peekFirst(); + if (fromLoopExitDuplication != null && !fromLoopExitDuplication.isEmpty()) { + return fromLoopExitDuplication.removeFirst(); } - if (nextIterationsFromUnrolling != null && !nextIterationsFromUnrolling.isEmpty()) { - return remove ? nextIterationsFromUnrolling.removeFirst() : nextIterationsFromUnrolling.peekFirst(); + if (fromUnrolling != null && !fromUnrolling.isEmpty()) { + return fromUnrolling.removeFirst(); } return null; } } - protected static class LoopExplosionState { - public final FrameState state; - public final MergeNode merge; - public final int hashCode; + protected static final class LoopExplosionKey { + public final int bci; + public final List values; + private final int hashCode; - protected LoopExplosionState(FrameState state, MergeNode merge) { - this.state = state; - this.merge = merge; + protected LoopExplosionKey(int bci, List values) { + this.bci = bci; + this.values = values; - int h = 0; - for (ValueNode value : state.values()) { + int h = bci; + for (ValueNode value : values) { if (value == null) { h = h * 31 + 1234; } else { @@ -546,21 +520,21 @@ protected LoopExplosionState(FrameState state, MergeNode merge) { @Override public boolean equals(Object obj) { - if (!(obj instanceof LoopExplosionState other)) { + if (!(obj instanceof LoopExplosionKey other)) { return false; } - // Check the hash code first to avoid iterating the frame states. + // Check the hash code first to avoid iterating the values. if (hashCode != other.hashCode) { return false; } - final FrameState thisState = state; - final FrameState otherState = other.state; - assert thisState.outerFrameState() == otherState.outerFrameState() : Assertions.errorMessage(thisState, thisState.outerFrameState(), otherState, otherState.outerFrameState()); + if (this.bci != other.bci) { + return false; + } - final NodeInputList thisValues = thisState.values(); - final NodeInputList otherValues = otherState.values(); + final List thisValues = this.values; + final List otherValues = other.values; final int size = thisValues.size(); if (size != otherValues.size()) { @@ -583,6 +557,23 @@ public boolean equals(Object obj) { public int hashCode() { return hashCode; } + + @Override + public String toString() { + return bci + "@[" + values.stream().map(v -> v.toString()).collect(Collectors.joining(", ")) + "]"; + } + } + + protected static final class LoopExplosionState { + public final FrameState state; + public final MergeNode merge; + public final int loopDepth; + + protected LoopExplosionState(FrameState state, MergeNode merge, int loopDepth) { + this.state = state; + this.merge = merge; + this.loopDepth = loopDepth; + } } protected static class InvokableData { @@ -652,6 +643,8 @@ public GraphDecoder(Architecture architecture, StructuredGraph graph) { reusableFloatingNodes = EconomicMap.create(Equivalence.IDENTITY); } + // #region Main decoder loop + public final void decode(EncodedGraph encodedGraph) { decode(encodedGraph, null); } @@ -660,26 +653,25 @@ public final void decode(EncodedGraph encodedGraph) { public final void decode(EncodedGraph encodedGraph, Iterable nodeReferences) { try (DebugContext.Scope scope = debug.scope("GraphDecoder", graph)) { recordGraphElements(encodedGraph); - MethodScope methodScope = new MethodScope(null, graph, encodedGraph, LoopExplosionPlugin.LoopExplosionKind.NONE); - LoopScope loopScope = createInitialLoopScope(methodScope, null); - decode(loopScope); - cleanupGraph(methodScope); - assert graph.verify(); + MethodScope rootMethodScope = createRootMethodScope(encodedGraph); + LoopScope initialLoopScope = createInitialLoopScope(rootMethodScope, null); + rootMethodScope.currentLoopScope = initialLoopScope; + doDecode(rootMethodScope); + cleanupGraph(rootMethodScope); + assert graph.verify(); if (nodeReferences != null) { for (var nodeReference : nodeReferences) { if (nodeReference.orderId < 0) { throw GraalError.shouldNotReachHere("EncodeNodeReference is not in 'encoded' state"); // ExcludeFromJacocoGeneratedReport } - nodeReference.node = loopScope.getNode(nodeReference.orderId); + nodeReference.node = initialLoopScope.getNode(nodeReference.orderId); if (nodeReference.node == null || !nodeReference.node.isAlive()) { throw GraalError.shouldNotReachHere("Could not decode the EncodedNodeReference"); // ExcludeFromJacocoGeneratedReport } nodeReference.orderId = EncodedGraph.EncodedNodeReference.DECODED; } } - - graph.maybeMarkUnsafeAccess(encodedGraph); } catch (Throwable ex) { debug.handle(ex); } @@ -704,6 +696,10 @@ protected void recordGraphElements(EncodedGraph encodedGraph) { graph.maybeMarkUnsafeAccess(encodedGraph); } + private MethodScope createRootMethodScope(EncodedGraph encodedGraph) { + return new MethodScope(null, graph, encodedGraph, LoopExplosionPlugin.LoopExplosionKind.NONE); + } + protected final LoopScope createInitialLoopScope(MethodScope methodScope, FixedWithNextNode startNode) { LoopScope loopScope = new LoopScope(methodScope); FixedNode firstNode; @@ -727,55 +723,74 @@ protected final LoopScope createInitialLoopScope(MethodScope methodScope, FixedW } @SuppressWarnings("try") - protected final void decode(LoopScope initialLoopScope) { - initialLoopScope.methodScope.replaceLogsForDecodedGraph(); + protected final void doDecode(MethodScope rootMethodScope) { + if (rootMethodScope.inliningLog != null) { + graph.setInliningLog(rootMethodScope.inliningLog); + } + if (rootMethodScope.optimizationLog != null) { + graph.setOptimizationLog(rootMethodScope.optimizationLog); + } try (InliningLog.UpdateScope updateScope = InliningLog.openDefaultUpdateScope(graph.getInliningLog())) { - LoopScope loopScope = initialLoopScope; + MethodScope methodScope = rootMethodScope; /* Process (inlined) methods. */ - while (loopScope != null) { - MethodScope methodScope = loopScope.methodScope; - - /* Process loops of method. */ - while (loopScope != null) { - /* Process nodes of loop. */ - while (!loopScope.nodesToProcess.isEmpty()) { - loopScope = processNextNode(methodScope, loopScope); - methodScope = loopScope.methodScope; - /* - * We can have entered a new loop, and we can have entered a new inlined - * method. - */ - } - - /* Finished with a loop. */ - if (loopScope.hasIterationsToProcess()) { - loopScope = loopScope.getNextIterationToProcess(true); - } else { - propagateCreatedNodes(loopScope); - loopScope = loopScope.outer; + while (methodScope != null) { + MethodScope newMethodScope = processMethodScope(methodScope); + /* + * We can have entered a new inlined method, or finished inlining. + */ + if (newMethodScope == null) { + /* + * Finished with an inlined method. Perform end-of-method cleanup tasks. + */ + finishMethod(methodScope); - if (loopScope == null) { - // finished all loops of a method - afterMethodScope(methodScope); - } - } + /* continue with the caller */ + methodScope = methodScope.caller; + } else { + methodScope = newMethodScope; } + } + } + } + private MethodScope processMethodScope(MethodScope methodScope) { + while (methodScope.currentLoopScope != null) { + /* Process nodes of loop. */ + while (!methodScope.currentLoopScope.nodesToProcess.isEmpty()) { + // TODO(GR-71752): use MethodScope instead of LoopScope here + LoopScope newLoopScope = processNextNode(methodScope, methodScope.currentLoopScope); /* - * Finished with an inlined method. Perform end-of-method cleanup tasks. + * We can have entered a new loop, and we can have entered a new inlined method. */ - if (methodScope.loopExplosion.mergeLoops()) { - LoopDetector loopDetector = new LoopDetector(graph, methodScope); - loopDetector.run(); - } - if (methodScope.isInlinedMethod()) { - finishInlining(methodScope); + if (newLoopScope.methodScope != methodScope) { + newLoopScope.methodScope.currentLoopScope = newLoopScope; + return newLoopScope.methodScope; + } else { + methodScope.currentLoopScope = newLoopScope; } + } - /* continue with the caller */ - loopScope = methodScope.callerLoopScope; + /* Finished with a loop. */ + LoopScope nextLoopScope = methodScope.currentLoopScope.nextIterations.takeNext(); + if (nextLoopScope != null) { + methodScope.currentLoopScope = nextLoopScope; + } else { + propagateCreatedNodes(methodScope.currentLoopScope); + methodScope.currentLoopScope = methodScope.currentLoopScope.outer; } } + return null; + } + + private void finishMethod(MethodScope methodScope) { + afterMethodScope(methodScope); + if (methodScope.loopExplosion.mergeLoops()) { + LoopDetector loopDetector = new LoopDetector(graph, methodScope); + loopDetector.run(); + } + if (methodScope.isInlinedMethod()) { + finishInlining(methodScope); + } } protected void afterMethodScope(@SuppressWarnings("unused") MethodScope methodScope) { @@ -797,6 +812,8 @@ private static void propagateCreatedNodes(LoopScope loopScope) { } } + // #endregion + public static final boolean DUMP_DURING_FIXED_NODE_PROCESSING = false; protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope) { @@ -852,8 +869,8 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope * loop exit of the inner loop. */ LoopScope outerScope = loopScope.outer; - int nextIterationNumber = outerScope.nextIterationFromLoopExitDuplication.isEmpty() ? outerScope.loopIteration + 1 - : outerScope.nextIterationFromLoopExitDuplication.getLast().loopIteration + 1; + int nextIterationNumber = outerScope.nextIterations.fromLoopExitDuplication.isEmpty() ? outerScope.loopIteration + 1 + : outerScope.nextIterations.fromLoopExitDuplication.getLast().loopIteration + 1; Node[] initialCreatedNodes = outerScope.initialCreatedNodes == null ? null : (methodScope.loopExplosion.mergeLoops() ? Arrays.copyOf(outerScope.initialCreatedNodes, outerScope.initialCreatedNodes.length) @@ -866,10 +883,8 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope successorAddScope = new LoopScope(methodScope, outerScope.outer, outerScope.loopDepth, nextIterationNumber, outerScope.loopBeginOrderId, LoopScopeTrigger.LOOP_EXIT_DUPLICATION, initialCreatedNodes, Arrays.copyOf(loopScope.initialCreatedNodes, loopScope.initialCreatedNodes.length), - outerScope.nextIterationFromLoopExitDuplication, - outerScope.nextIterationFromLoopEndDuplication, - outerScope.nextIterationsFromUnrolling, - outerScope.iterationStates, + outerScope.nextIterations, + outerScope.loopExplosionMergeKeySlots, false); checkLoopExplosionIteration(methodScope, successorAddScope); @@ -882,7 +897,7 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope successorAddScope.setNode(id, null); } - outerScope.nextIterationFromLoopExitDuplication.addLast(successorAddScope); + outerScope.nextIterations.fromLoopExitDuplication.addLast(successorAddScope); } else { successorAddScope = loopScope.outer; } @@ -907,14 +922,12 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope if (methodScope.loopExplosion.useExplosion()) { handleLoopExplosionBegin(methodScope, loopScope, (LoopBeginNode) node); } - } else if (node instanceof LoopExitNode) { if (methodScope.loopExplosion.useExplosion()) { handleLoopExplosionProxyNodes(methodScope, loopScope, successorAddScope, (LoopExitNode) node, nodeOrderId); } else { handleProxyNodes(methodScope, loopScope, (LoopExitNode) node); } - } else if (node instanceof MergeNode) { handleMergeNode(((MergeNode) node)); } else if (node instanceof AbstractEndNode) { @@ -942,25 +955,23 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope * Therefore, we create a correct outer loop iteration and check if there is * already one, if not we create it else we re-use it. */ - if (loopScope.nextIterationsFromUnrolling.isEmpty()) { + if (loopScope.nextIterations.fromUnrolling.isEmpty()) { // create it - int nextIterationNumber = loopScope.nextIterationsFromUnrolling.isEmpty() ? loopScope.loopIteration + 1 : loopScope.nextIterationsFromUnrolling.getLast().loopIteration + 1; + int nextIterationNumber = loopScope.nextIterations.fromUnrolling.isEmpty() ? loopScope.loopIteration + 1 : loopScope.nextIterations.fromUnrolling.getLast().loopIteration + 1; LoopScope outerLoopMergeScope = new LoopScope(methodScope, loopScope.outer, loopScope.loopDepth, nextIterationNumber, loopScope.loopBeginOrderId, LoopScopeTrigger.LOOP_BEGIN_UNROLLING, methodScope.loopExplosion.mergeLoops() ? Arrays.copyOf(loopScope.initialCreatedNodes, loopScope.initialCreatedNodes.length) : loopScope.initialCreatedNodes, Arrays.copyOf(loopScope.initialCreatedNodes, loopScope.initialCreatedNodes.length), - loopScope.nextIterationFromLoopExitDuplication, - loopScope.nextIterationFromLoopEndDuplication, - loopScope.nextIterationsFromUnrolling, - loopScope.iterationStates); + loopScope.nextIterations, + loopScope.loopExplosionMergeKeySlots); checkLoopExplosionIteration(methodScope, outerLoopMergeScope); - loopScope.nextIterationsFromUnrolling.addLast(outerLoopMergeScope); + loopScope.nextIterations.fromUnrolling.addLast(outerLoopMergeScope); registerNode(outerLoopMergeScope, loopScope.loopBeginOrderId, null, true, true); makeStubNode(methodScope, outerLoopMergeScope, loopScope.loopBeginOrderId); phiNodeScope = outerLoopMergeScope; } else { // re-use it - phiNodeScope = loopScope.nextIterationsFromUnrolling.getLast(); + phiNodeScope = loopScope.nextIterations.fromUnrolling.getLast(); } } else if (methodScope.loopExplosion.useExplosion() && node instanceof LoopEndNode) { @@ -969,9 +980,9 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope node.safeDelete(); node = replacementNode; LoopScopeTrigger trigger = handleLoopExplosionEnd(methodScope, loopScope); - Deque phiScope = loopScope.nextIterationsFromUnrolling; + Deque phiScope = loopScope.nextIterations.fromUnrolling; if (trigger == LoopScopeTrigger.LOOP_END_DUPLICATION) { - phiScope = loopScope.nextIterationFromLoopEndDuplication; + phiScope = loopScope.nextIterations.fromLoopEndDuplication; } phiNodeScope = phiScope.getLast(); } @@ -1007,10 +1018,8 @@ protected LoopScope processNextNode(MethodScope methodScope, LoopScope loopScope resultScope = new LoopScope(methodScope, loopScope, loopScope.loopDepth + 1, 0, mergeOrderId, LoopScopeTrigger.START, methodScope.loopExplosion.useExplosion() ? Arrays.copyOf(createdNodes, createdNodes.length) : null, methodScope.loopExplosion.useExplosion() ? new Node[createdNodes.length] : createdNodes, // - methodScope.loopExplosion.duplicateLoopExits() || methodScope.loopExplosion.mergeLoops() ? new ArrayDeque<>(2) : null, - methodScope.loopExplosion.duplicateLoopEnds() ? new ArrayDeque<>(2) : null, - methodScope.loopExplosion.unrollLoops() ? new ArrayDeque<>(2) : null, // - methodScope.loopExplosion.mergeLoops() ? EconomicMap.create(Equivalence.DEFAULT) : null); + new LoopIterationQueue(methodScope.loopExplosion), + null); phiInputScope = resultScope; phiNodeScope = resultScope; @@ -1131,9 +1140,16 @@ protected void handleLoopExplosionBegin(MethodScope methodScope, LoopScope loopS FrameState frameState = loopBegin.stateAfter(); if (methodScope.loopExplosion.mergeLoops()) { - LoopExplosionState queryState = new LoopExplosionState(frameState, null); - LoopExplosionState existingState = loopScope.iterationStates.get(queryState); + if (loopScope.trigger == LoopScopeTrigger.START) { + loopScope.loopExplosionMergeKeySlots = computeIndicesForMergeKey(loopScope, frameState); + } + LoopExplosionKey key = createLoopExplosionKey(loopScope, frameState); + LoopExplosionState existingState = methodScope.iterationStates.get(key); if (existingState != null) { + if (!frameStateEquals(frameState, existingState.state)) { + throw new PermanentBailoutException("Graal implementation restriction: Method with %s loop explosion has differing values in non-key locals", + LoopExplosionPlugin.LoopExplosionKind.MERGE_EXPLODE); + } loopBegin.replaceAtUsagesAndDelete(existingState.merge); successor.safeDelete(); for (EndNode predecessor : predecessors) { @@ -1152,7 +1168,7 @@ protected void handleLoopExplosionBegin(MethodScope methodScope, LoopScope loopS methodScope.loopExplosionMerges.add(merge); if (methodScope.loopExplosion.mergeLoops()) { - if (loopScope.iterationStates.size() == 0 && loopScope.loopDepth == 1) { + if (methodScope.iterationStates.size() == 0 && loopScope.loopDepth == 1) { if (methodScope.loopExplosionHead != null) { throw new PermanentBailoutException("Graal implementation restriction: Method with %s loop explosion must not have more than one top-level loop", LoopExplosionPlugin.LoopExplosionKind.MERGE_EXPLODE); @@ -1169,11 +1185,84 @@ protected void handleLoopExplosionBegin(MethodScope methodScope, LoopScope loopS } if (methodScope.loopExplosion.mergeLoops()) { - LoopExplosionState explosionState = new LoopExplosionState(frameState, merge); - loopScope.iterationStates.put(explosionState, explosionState); + LoopExplosionKey explosionKey = createLoopExplosionKey(loopScope, frameState); + LoopExplosionState explosionState = new LoopExplosionState(frameState, merge, loopScope.loopDepth); + methodScope.iterationStates.put(explosionKey, explosionState); } } + protected List computeIndicesForMergeKey(LoopScope loopScope, FrameState frameState) { + List mergeKey = null; + NodeInputList values = frameState.values(); + for (int i = 0; i < values.size(); i++) { + if (values.get(i) instanceof LoopExplosionKeyNode keyNode) { + if (mergeKey == null) { + mergeKey = new ArrayList<>(); + } + mergeKey.add(i); + + for (int j = 0; j < loopScope.createdNodes.length; j++) { + if (loopScope.createdNodes[j] == keyNode) { + loopScope.createdNodes[j] = keyNode.value; + } + } + + if (loopScope.initialCreatedNodes != null) { + for (int j = 0; j < loopScope.initialCreatedNodes.length; j++) { + if (loopScope.initialCreatedNodes[j] == keyNode) { + loopScope.initialCreatedNodes[j] = keyNode.value; + } + } + } + + keyNode.replaceAndDelete(keyNode.value); + } + } + if (mergeKey != null && mergeKey.size() > 1) { + throw new PermanentBailoutException("Graal implementation restriction: Method with %s loop explosion has more than one specified merge key local", + LoopExplosionPlugin.LoopExplosionKind.MERGE_EXPLODE); + } + return mergeKey; + } + + protected LoopExplosionKey createLoopExplosionKey(LoopScope loopScope, FrameState frameState) { + List values; + if (loopScope.loopExplosionMergeKeySlots != null) { + values = new ArrayList<>(loopScope.loopExplosionMergeKeySlots.size()); + for (int i : loopScope.loopExplosionMergeKeySlots) { + ValueNode node = frameState.values().get(i); + if (!node.isConstant() || node.asJavaConstant().getJavaKind() != JavaKind.Int) { + throw new PermanentBailoutException("Graal implementation restriction: merge keys must partial evaluate to int constants at the loop header. %s", + node); + } + values.add(node); + } + } else { + /** + * In case there are no marked variables, we fall back to using the entire frame state + * as the key. This ensures compatibility with old code that doesn't use explicit keys + * yet. + */ + values = frameState.values().snapshot(); + } + return new LoopExplosionKey(frameState.bci, values); + } + + private static boolean frameStateEquals(FrameState a, FrameState b) { + assert b.outerFrameState() == a.outerFrameState() : Assertions.errorMessage(b, b.outerFrameState(), a, a.outerFrameState()); + + Iterator aIter = b.values().iterator(); + Iterator bIter = a.values().iterator(); + while (aIter.hasNext() && bIter.hasNext()) { + ValueNode aValue = aIter.next(); + ValueNode bValue = bIter.next(); + if (aValue != bValue) { + return false; + } + } + return aIter.hasNext() == bIter.hasNext(); + } + /** * Hook for subclasses. * @@ -1198,14 +1287,14 @@ protected LoopScopeTrigger handleLoopExplosionEnd(MethodScope methodScope, LoopS * end. */ trigger = LoopScopeTrigger.LOOP_END_DUPLICATION; - nextIterations = loopScope.nextIterationFromLoopEndDuplication; - } else if (loopScope.nextIterationsFromUnrolling.isEmpty()) { + nextIterations = loopScope.nextIterations.fromLoopEndDuplication; + } else if (loopScope.nextIterations.fromUnrolling.isEmpty()) { /* * Regular loop unrolling, i.e., we reach a loop end node of a loop that should be * unrolled: We create a new successor scope. */ trigger = LoopScopeTrigger.LOOP_BEGIN_UNROLLING; - nextIterations = loopScope.nextIterationsFromUnrolling; + nextIterations = loopScope.nextIterations.fromUnrolling; } if (trigger != null) { final LoopScope nextIterationScope = createNextLoopIterationScope(methodScope, loopScope, trigger, nextIterations); @@ -1249,10 +1338,8 @@ private static LoopScope createNextLoopIterationScope(MethodScope methodScope, L return new LoopScope(methodScope, loopScope.outer, loopScope.loopDepth, nextIterationNumber, loopScope.loopBeginOrderId, trigger, initialCreatedNodes, createdNodes, - loopScope.nextIterationFromLoopExitDuplication, - loopScope.nextIterationFromLoopEndDuplication, - loopScope.nextIterationsFromUnrolling, - loopScope.iterationStates); + loopScope.nextIterations, + loopScope.loopExplosionMergeKeySlots); } /** @@ -1772,6 +1859,8 @@ protected Node handleFloatingNodeAfterAdd(MethodScope methodScope, LoopScope loo return node; } + // #region Reading from encoded graph + /** * Process successor edges of a node. We create the successor nodes so that we can fill the * successor list, but no properties or edges are loaded yet. That is done when the successor is @@ -1905,6 +1994,10 @@ protected Object readObject(MethodScope methodScope) { return methodScope.encodedGraph.getObject(methodScope.reader.getUVInt()); } + // #endregion + + // #region Cleanup + /** * Removes unnecessary nodes from the graph after decoding. * @@ -1914,7 +2007,7 @@ protected void cleanupGraph(MethodScope rootMethodScope) { assert verifyEdges(); } - protected boolean verifyEdges() { + private boolean verifyEdges() { for (Node node : graph.getNodes()) { assert node.isAlive(); for (Node i : node.inputs()) { @@ -1937,6 +2030,8 @@ protected boolean verifyEdges() { } return true; } + + // #endregion } class LoopDetector implements Runnable { @@ -2256,7 +2351,7 @@ private void findLoopExits(Loop loop) { * // outerLoopContinueCode that uses values proxied inside the loop * * - * We would produce two loop exits that merge booth on the outerLoopContinueCode. + * We would produce two loop exits that merge both on the outerLoopContinueCode. * This would require the generation of complex phi and proxy constructs, thus we include the merge inside the * loop if we find a subsequent loop explosion merge. */ @@ -2286,27 +2381,22 @@ private void findLoopExits(Loop loop) { // we found a shared merge as outlined above if (mergesToRemove.size() > 0) { assert merges.size() < loop.exits.size() : Assertions.errorMessage(merges, loop, loop.exits); - outer: for (MergeNode merge : mergesToRemove) { + for (MergeNode merge : mergesToRemove) { FixedNode current = merge; - while (current != null) { - if (current instanceof FixedWithNextNode) { - current = ((FixedWithNextNode) current).next(); - continue; - } - if (current instanceof EndNode && methodScope.loopExplosionMerges.contains(((EndNode) current).merge())) { - // we found the place for the loop exit introduction since the subsequent - // merge has a frame state - loop.exits.removeIf(x -> x.merge() == merge); - loop.exits.add((EndNode) current); - break; - } - /* - * No next merge was found, this can only mean no immediate unroll happend next, - * i.e., there is no subsequent iteration of any loop exploded directly after, - * thus no loop exit possible. - */ - continue outer; + while (current instanceof FixedWithNextNode c) { + current = c.next(); + } + if (current instanceof EndNode end && methodScope.loopExplosionMerges.contains(end.merge())) { + // we found the place for the loop exit introduction since the subsequent + // merge has a frame state + loop.exits.removeIf(x -> x.merge() == merge); + loop.exits.add(end); } + /* + * Else, no next merge was found, this can only mean no immediate unroll happend + * next, i.e., there is no subsequent iteration of any loop exploded directly after, + * thus no loop exit possible. + */ } } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/InliningLogCodec.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/InliningLogCodec.java index 598b5bc7083c..e55e946a8a6a 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/InliningLogCodec.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/InliningLogCodec.java @@ -206,11 +206,10 @@ private InliningLog decode(Object encodedObject) { * @param orderId the order ID of the registered node */ public void registerNode(InliningLog inliningLog, Node node, int orderId) { - if (!(node instanceof Invokable)) { + if (!(node instanceof Invokable invokable)) { return; } assert orderIdToCallsite != null : "registerNode should be called after decode"; - Invokable invokable = (Invokable) node; if (inliningLog.containsLeafCallsite(invokable)) { return; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopExplosionKeyNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopExplosionKeyNode.java new file mode 100644 index 000000000000..100bddfc6f5b --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopExplosionKeyNode.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_0; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_0; + +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.calc.FloatingNode; +import jdk.graal.compiler.nodes.spi.Canonicalizable; +import jdk.graal.compiler.nodes.spi.CanonicalizerTool; + +@NodeInfo(nameTemplate = "LoopExplosionKey", cycles = CYCLES_0, size = SIZE_0) +public final class LoopExplosionKeyNode extends FloatingNode implements Canonicalizable { + public static final NodeClass TYPE = NodeClass.create(LoopExplosionKeyNode.class); + + @Input ValueNode value; + + public LoopExplosionKeyNode(ValueNode value) { + super(TYPE, value.stamp); + this.value = value; + } + + @Override + public Node canonical(CanonicalizerTool tool) { + return this; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java index b7032d3c8e26..3f70de971127 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java @@ -199,9 +199,9 @@ protected class PEMethodScope extends MethodScope { public ExceptionPlaceholderNode exceptionPlaceholderNode; protected NodeSourcePosition callerBytecodePosition; - protected PEMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, + protected PEMethodScope(StructuredGraph targetGraph, PEMethodScope caller, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, int inliningDepth, ValueNode[] arguments) { - super(callerLoopScope, targetGraph, encodedGraph, loopExplosionKind(method, loopExplosionPlugin)); + super(caller, targetGraph, encodedGraph, loopExplosionKind(method, loopExplosionPlugin)); this.caller = caller; this.method = method; @@ -215,11 +215,6 @@ protected PEMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopS } } - @Override - public boolean isInlinedMethod() { - return caller != null; - } - public ValueNode[] getArguments() { return arguments; } @@ -563,9 +558,9 @@ public void push(JavaKind kind, ValueNode value) { @Override public void setStateAfter(StateSplit stateSplit) { - Node stateAfter = decodeFloatingNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId); + Node stateAfter = decodeFloatingNode(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData.stateAfterOrderId); getGraph().add(stateAfter); - FrameState fs = (FrameState) handleFloatingNodeAfterAdd(methodScope.caller, methodScope.callerLoopScope, stateAfter); + FrameState fs = (FrameState) handleFloatingNodeAfterAdd(methodScope.caller, methodScope.caller.currentLoopScope, stateAfter); stateSplit.setStateAfter(fs); } @@ -642,7 +637,7 @@ private void updateLastInstruction(T v) { exceptionEdgeConsumed = true; WithExceptionNode withExceptionNode = (WithExceptionNode) fixedNode; if (withExceptionNode.exceptionEdge() == null) { - ExceptionObjectNode exceptionEdge = (ExceptionObjectNode) makeStubNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.exceptionOrderId); + ExceptionObjectNode exceptionEdge = (ExceptionObjectNode) makeStubNode(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData.exceptionOrderId); withExceptionNode.setExceptionEdge(exceptionEdge); } if (withExceptionNode.next() == null) { @@ -683,7 +678,7 @@ public void handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType) invokeConsumed = true; exceptionEdgeConsumed = true; - appendInvoke(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData, callTarget); + appendInvoke(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData, callTarget); lastInstr.setNext(invoke.asFixedNode()); if (invoke instanceof InvokeWithExceptionNode) { @@ -718,8 +713,8 @@ public AbstractBeginNode genExplicitExceptionEdge(BytecodeExceptionNode.Bytecode methodScope.exceptionPlaceholderNode.replaceAtUsagesAndDelete(exceptionNode); - registerNode(methodScope.callerLoopScope, methodScope.invokeData.exceptionOrderId, exceptionNode, true, false); - exceptionNode.setNext(makeStubNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.exceptionNextOrderId)); + registerNode(methodScope.caller.currentLoopScope, methodScope.invokeData.exceptionOrderId, exceptionNode, true, false); + exceptionNode.setNext(makeStubNode(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData.exceptionNextOrderId)); } return BeginNode.begin(exceptionNode); @@ -904,8 +899,10 @@ public void decode(ResolvedJavaMethod method) { try (DebugContext.Scope scope = debug.scope("PEGraphDecode", graph)) { EncodedGraph encodedGraph = lookupEncodedGraph(method, null); recordGraphElements(encodedGraph); - PEMethodScope methodScope = createMethodScope(graph, null, null, encodedGraph, method, null, 0, null); - decode(createInitialLoopScope(methodScope, null)); + PEMethodScope methodScope = createMethodScope(graph, null, encodedGraph, method, null, 0, null); + LoopScope initialLoopScope = createInitialLoopScope(methodScope, null); + methodScope.currentLoopScope = initialLoopScope; + doDecode(methodScope); debug.dump(DebugContext.VERBOSE_LEVEL, graph, "Before graph cleanup"); cleanupGraph(methodScope); @@ -924,9 +921,9 @@ public void decode(ResolvedJavaMethod method) { } } - protected PEMethodScope createMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, + protected PEMethodScope createMethodScope(StructuredGraph targetGraph, PEMethodScope caller, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, int inliningDepth, ValueNode[] arguments) { - return new PEMethodScope(targetGraph, caller, callerLoopScope, encodedGraph, method, invokeData, inliningDepth, arguments); + return new PEMethodScope(targetGraph, caller, encodedGraph, method, invokeData, inliningDepth, arguments); } @Override @@ -1136,7 +1133,7 @@ protected boolean tryInvocationPlugin(PEMethodScope methodScope, LoopScope loopS */ invoke.asNode().replaceAtPredecessor(null); - PEMethodScope inlineScope = createMethodScope(graph, methodScope, loopScope, null, targetMethod, invokeData, methodScope.inliningDepth + 1, arguments); + PEMethodScope inlineScope = createMethodScope(graph, methodScope, null, targetMethod, invokeData, methodScope.inliningDepth + 1, arguments); JavaType returnType = targetMethod.getSignature().getReturnType(methodScope.method.getDeclaringClass()); PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(inlineScope, invokePredecessor, callTarget.invokeKind(), returnType, true, false); @@ -1245,7 +1242,7 @@ protected LoopScope doInline(PEMethodScope methodScope, LoopScope loopScope, Inv invokeData.invokePredecessor = predecessor; invokeNode.replaceAtPredecessor(null); - PEMethodScope inlineScope = createMethodScope(graph, methodScope, loopScope, graphToInline, inlineMethod, invokeData, methodScope.inliningDepth + 1, arguments); + PEMethodScope inlineScope = createMethodScope(graph, methodScope, graphToInline, inlineMethod, invokeData, methodScope.inliningDepth + 1, arguments); if (!inlineMethod.isStatic()) { if (StampTool.isPointerAlwaysNull(arguments[0])) { @@ -1323,7 +1320,7 @@ protected void finishInlining(MethodScope is) { PEMethodScope inlineScope = (PEMethodScope) is; ResolvedJavaMethod inlineMethod = inlineScope.method; PEMethodScope methodScope = inlineScope.caller; - LoopScope loopScope = inlineScope.callerLoopScope; + LoopScope loopScope = inlineScope.caller.currentLoopScope; InvokeData invokeData = inlineScope.invokeData; Invoke invoke = invokeData.invoke; FixedNode invokeNode = invoke.asFixedNode(); @@ -1701,7 +1698,7 @@ protected void ensureOuterStateDecoded(PEMethodScope methodScope) { if (methodScope.outerState == null && methodScope.caller != null) { FrameState stateAtReturn = methodScope.invokeData.invoke.stateAfter(); if (stateAtReturn == null) { - stateAtReturn = (FrameState) decodeFloatingNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId); + stateAtReturn = (FrameState) decodeFloatingNode(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData.stateAfterOrderId); } JavaKind invokeReturnKind = methodScope.invokeData.invoke.asNode().getStackKind(); @@ -1735,7 +1732,7 @@ protected boolean shouldOmitIntermediateMethodInStates(@SuppressWarnings("unused protected void ensureStateAfterDecoded(PEMethodScope methodScope) { if (methodScope.invokeData.invoke.stateAfter() == null) { - methodScope.invokeData.invoke.setStateAfter((FrameState) ensureNodeCreated(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId)); + methodScope.invokeData.invoke.setStateAfter((FrameState) ensureNodeCreated(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData.stateAfterOrderId)); } } @@ -1745,8 +1742,8 @@ protected void ensureExceptionStateDecoded(PEMethodScope methodScope) { assert methodScope.exceptionPlaceholderNode == null; methodScope.exceptionPlaceholderNode = graph.add(new ExceptionPlaceholderNode()); - registerNode(methodScope.callerLoopScope, methodScope.invokeData.exceptionOrderId, methodScope.exceptionPlaceholderNode, false, false); - FrameState exceptionState = (FrameState) ensureNodeCreated(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.exceptionStateOrderId); + registerNode(methodScope.caller.currentLoopScope, methodScope.invokeData.exceptionOrderId, methodScope.exceptionPlaceholderNode, false, false); + FrameState exceptionState = (FrameState) ensureNodeCreated(methodScope.caller, methodScope.caller.currentLoopScope, methodScope.invokeData.exceptionStateOrderId); if (exceptionState.outerFrameState() == null && methodScope.caller != null) { ensureOuterStateDecoded(methodScope.caller); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/TruffleTierContext.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/TruffleTierContext.java index c223f48fcda8..0b709a98f315 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/TruffleTierContext.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/TruffleTierContext.java @@ -109,7 +109,8 @@ public static TruffleTierContext createInitialContext(PartialEvaluator partialEv DebugContext debug, TruffleCompilable compilable, CompilationIdentifier compilationId, SpeculationLog log, - TruffleCompilationTask task, PerformanceInformationHandler handler) { + TruffleCompilationTask task, + PerformanceInformationHandler handler) { boolean readyForCompilation = compilable.prepareForCompilation(true, task.tier(), !task.hasNextTier()); if (!readyForCompilation) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java index 9cd6e978cbf7..823df107cd8c 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/substitutions/TruffleGraphBuilderPlugins.java @@ -63,6 +63,7 @@ import jdk.graal.compiler.nodes.FrameState; import jdk.graal.compiler.nodes.InvokeNode; import jdk.graal.compiler.nodes.LogicNode; +import jdk.graal.compiler.nodes.LoopExplosionKeyNode; import jdk.graal.compiler.nodes.NamedLocationIdentity; import jdk.graal.compiler.nodes.NodeView; import jdk.graal.compiler.nodes.PiArrayNode; @@ -514,6 +515,22 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec return true; } }); + + r.register(new RequiredInvocationPlugin("mergeExplodeKey", int.class) { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { + if (canDelayIntrinsification) { + /* + * We make sure to keep the original call while parsing the method so we don't + * lose the node when encoding the graph. At the time of decoding, + * `canDelayIntrinsification` will be false, so the actual node is created. + */ + return false; + } + b.push(JavaKind.Int, b.add(new LoopExplosionKeyNode(value))); + return true; + } + }); } private static Class getJavaClass(JavaKind kind) { diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java index 01df9528ded8..e7a632824d7d 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java @@ -755,6 +755,7 @@ private Object executeBodyFromBCI(VirtualFrame frame, int startBCI, int startTop livenessAnalysis.onStart(frame, skipLivenessActions); } + curBCI = CompilerDirectives.mergeExplodeKey(curBCI); loop: while (true) { final int curOpcode = bs.opcode(curBCI); EXECUTED_BYTECODES_COUNT.inc(); diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java index beb2718761a8..d59d5ca56df6 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java @@ -76,9 +76,9 @@ public class InlineBeforeAnalysisMethodScope extends PEMethodScope { */ private final EconomicSet encodedGraphs; - InlineBeforeAnalysisMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, AnalysisMethod method, + InlineBeforeAnalysisMethodScope(StructuredGraph targetGraph, PEMethodScope caller, EncodedGraph encodedGraph, AnalysisMethod method, InvokeData invokeData, int inliningDepth, ValueNode[] arguments) { - super(targetGraph, caller, callerLoopScope, encodedGraph, method, invokeData, inliningDepth, arguments); + super(targetGraph, caller, encodedGraph, method, invokeData, inliningDepth, arguments); if (caller == null) { /* The root method that we are decoding, i.e., inlining into. */ @@ -168,9 +168,9 @@ protected void cleanupGraph(MethodScope ms) { } @Override - protected PEMethodScope createMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, + protected PEMethodScope createMethodScope(StructuredGraph targetGraph, PEMethodScope caller, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, int inliningDepth, ValueNode[] arguments) { - return new InlineBeforeAnalysisMethodScope(targetGraph, caller, callerLoopScope, encodedGraph, (AnalysisMethod) method, invokeData, inliningDepth, arguments); + return new InlineBeforeAnalysisMethodScope(targetGraph, caller, encodedGraph, (AnalysisMethod) method, invokeData, inliningDepth, arguments); } @Override @@ -302,7 +302,7 @@ protected void recordGraphElements(EncodedGraph encodedGraph) { protected void finishInlining(MethodScope is) { InlineBeforeAnalysisMethodScope inlineScope = cast(is); InlineBeforeAnalysisMethodScope callerScope = cast(inlineScope.caller); - LoopScope callerLoopScope = inlineScope.callerLoopScope; + LoopScope callerLoopScope = inlineScope.caller.currentLoopScope; InvokeData invokeData = inlineScope.invokeData; if (inlineScope.inliningAborted) { diff --git a/truffle/CHANGELOG.md b/truffle/CHANGELOG.md index 9396ea965328..dedeb1a6e98d 100644 --- a/truffle/CHANGELOG.md +++ b/truffle/CHANGELOG.md @@ -61,9 +61,9 @@ This changelog summarizes major changes between Truffle versions relevant to lan * GR-71088 Added `CompilerDirectives.EarlyEscapeAnalysis` annotation that runs partial escape analysis early before partial evaluation enabling partial-evaluation-constant scalar replacements. * GR-71870 Truffle DSL no longer supports mixed exclusive and shared inlined caches. Sharing will now be disabled if mixing was used. To resolve the new warnings it is typically necessary to use either `@Exclusive` or `@Shared` for all caches. * GR-71887: Bytecode DSL: Added a `ClearLocal` operation for fast clearing of local values. -* GR-71088 Added `CompilerDirectives.EarlyEscapeAnalysis` annotation that runs partial escape analysis early before partial evaluation enabling partial-evaluation-constant scalar replacements. * GR-71402: Added `InteropLibrary#isHostObject` and `InteropLibrary#asHostObject` for accessing the Java host-object representation of a Truffle guest object. Deprecated `Env#isHostObject`, `Env#isHostException`, `Env#isHostFunction`, `Env#isHostSymbol`, `Env#asHostObject`, and `Env#asHostException` in favor of the new InteropLibrary messages. * GR-71402: Added `InteropLibrary#hasStaticScope` and `InteropLibrary#getStaticScope` returning the static scope representing static or class-level members associated with the given meta object. +* GR-71613: Added `CompilerDirectives.mergeExplodeKey` method for explicitly marking a local variable as a key for `@ExplodeLoop(MERGE_EXPLODE)` methods. It is recommended to migrate all merge exploded loops to use this method to catch unintended graph size explosions. ## Version 25.0 diff --git a/truffle/src/com.oracle.truffle.api/snapshot.sigtest b/truffle/src/com.oracle.truffle.api/snapshot.sigtest index 42de8dbb4c2f..0fbd10d56846 100644 --- a/truffle/src/com.oracle.truffle.api/snapshot.sigtest +++ b/truffle/src/com.oracle.truffle.api/snapshot.sigtest @@ -84,6 +84,7 @@ meth public static void interpreterOnly(java.lang.Runnable) meth public static void materialize(java.lang.Object) meth public static void transferToInterpreter() meth public static void transferToInterpreterAndInvalidate() +meth public static int mergeExplodeKey(int) supr java.lang.Object hcls ShouldNotReachHere diff --git a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/CompilerDirectives.java b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/CompilerDirectives.java index cf45d4168b74..7996ea31e3e4 100644 --- a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/CompilerDirectives.java +++ b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/CompilerDirectives.java @@ -790,4 +790,33 @@ static final class ShouldNotReachHere extends AssertionError { } } + + /** + * Marks a local variable as part of the key used for merging exploded loop iterations. This + * method must be used directly before a loop and the return value has to be assigned back to + * the variable. + * + *
+     * int bci = 0;
+     * // ...
+     * bci = CompilerDirectives.mergeExplodeKey(bci);
+     * while (bci != END_BCI) {
+     *     // ...
+     * }
+     * 
+ * + * Only a single variable can currently be annotated with this method, and at every iteration, + * the value must be a constant integer. Other variables must have the same value at every + * iteration where the key matches (e.g., partial evaluate to the same constant or remain + * unchanged). + * + * @param i the variable that should be used as the merge key. + * @return the unchanged value + * + * @since 25.1 + */ + public static int mergeExplodeKey(int i) { + return i; + } + } diff --git a/truffle/src/com.oracle.truffle.compiler/src/com/oracle/truffle/compiler/TruffleCompilable.java b/truffle/src/com.oracle.truffle.compiler/src/com/oracle/truffle/compiler/TruffleCompilable.java index 39e71339a5bf..25ae8ab6ed16 100644 --- a/truffle/src/com.oracle.truffle.compiler/src/com/oracle/truffle/compiler/TruffleCompilable.java +++ b/truffle/src/com.oracle.truffle.compiler/src/com/oracle/truffle/compiler/TruffleCompilable.java @@ -227,8 +227,6 @@ default int getSuccessfulCompilationCount() { * mean it will always be inlined, as inlining is a complex process that takes many factors into * account. If this method returns false, it will never be inlined. This typically * means that compilation with this compilable as the root has failed. - * - * */ default boolean canBeInlined() { return true;