src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java
author dlong
Thu, 15 Nov 2018 09:04:07 -0800
changeset 52578 7dd81e82d083
parent 50858 2d3e99a72541
child 52910 583fd71c47d6
permissions -rw-r--r--
8210777: Update Graal Reviewed-by: kvn
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     1
/*
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     2
 * Copyright (c) 2012, 2012, Oracle and/or its affiliates. All rights reserved.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     4
 *
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     7
 * published by the Free Software Foundation.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     8
 *
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    13
 * accompanied this code).
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    14
 *
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    18
 *
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    21
 * questions.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    22
 */
50858
2d3e99a72541 8205824: Update Graal
never
parents: 50330
diff changeset
    23
2d3e99a72541 8205824: Update Graal
never
parents: 50330
diff changeset
    24
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    25
package org.graalvm.compiler.loop.phases;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    26
46807
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
    27
import static org.graalvm.compiler.core.common.GraalOptions.MaximumDesiredSize;
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
    28
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
    29
import java.util.ArrayList;
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
    30
import java.util.Iterator;
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
    31
import java.util.List;
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
    32
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
    33
import jdk.internal.vm.compiler.collections.EconomicMap;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    34
import org.graalvm.compiler.core.common.RetryableBailoutException;
48861
47f19ff9903c 8194819: Update Graal
iveresov
parents: 47216
diff changeset
    35
import org.graalvm.compiler.core.common.calc.CanonicalCondition;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    36
import org.graalvm.compiler.debug.DebugContext;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    37
import org.graalvm.compiler.graph.Graph.Mark;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    38
import org.graalvm.compiler.graph.Node;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    39
import org.graalvm.compiler.graph.Position;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    40
import org.graalvm.compiler.loop.CountedLoopInfo;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    41
import org.graalvm.compiler.loop.InductionVariable.Direction;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    42
import org.graalvm.compiler.loop.LoopEx;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    43
import org.graalvm.compiler.loop.LoopFragmentInside;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    44
import org.graalvm.compiler.loop.LoopFragmentWhole;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    45
import org.graalvm.compiler.nodeinfo.InputType;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    46
import org.graalvm.compiler.nodes.AbstractBeginNode;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    47
import org.graalvm.compiler.nodes.AbstractEndNode;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    48
import org.graalvm.compiler.nodes.AbstractMergeNode;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    49
import org.graalvm.compiler.nodes.BeginNode;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    50
import org.graalvm.compiler.nodes.ControlSplitNode;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    51
import org.graalvm.compiler.nodes.EndNode;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    52
import org.graalvm.compiler.nodes.FixedNode;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    53
import org.graalvm.compiler.nodes.FixedWithNextNode;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    54
import org.graalvm.compiler.nodes.IfNode;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    55
import org.graalvm.compiler.nodes.LogicNode;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    56
import org.graalvm.compiler.nodes.LoopBeginNode;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    57
import org.graalvm.compiler.nodes.LoopExitNode;
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
    58
import org.graalvm.compiler.nodes.NodeView;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    59
import org.graalvm.compiler.nodes.PhiNode;
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
    60
import org.graalvm.compiler.nodes.SafepointNode;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    61
import org.graalvm.compiler.nodes.StructuredGraph;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    62
import org.graalvm.compiler.nodes.ValueNode;
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
    63
import org.graalvm.compiler.nodes.calc.AddNode;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    64
import org.graalvm.compiler.nodes.calc.CompareNode;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    65
import org.graalvm.compiler.nodes.calc.ConditionalNode;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    66
import org.graalvm.compiler.nodes.calc.IntegerLessThanNode;
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
    67
import org.graalvm.compiler.nodes.extended.OpaqueNode;
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    68
import org.graalvm.compiler.nodes.extended.SwitchNode;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    69
import org.graalvm.compiler.phases.common.CanonicalizerPhase;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    70
import org.graalvm.compiler.phases.tiers.PhaseContext;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    71
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    72
public abstract class LoopTransformations {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    73
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    74
    private LoopTransformations() {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    75
        // does not need to be instantiated
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    76
    }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    77
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    78
    public static void peel(LoopEx loop) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    79
        loop.inside().duplicate().insertBefore(loop);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    80
        loop.loopBegin().setLoopFrequency(Math.max(0.0, loop.loopBegin().loopFrequency() - 1));
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    81
    }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    82
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    83
    public static void fullUnroll(LoopEx loop, PhaseContext context, CanonicalizerPhase canonicalizer) {
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
    84
        // assert loop.isCounted(); //TODO (gd) strengthen : counted with known trip count
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    85
        LoopBeginNode loopBegin = loop.loopBegin();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    86
        StructuredGraph graph = loopBegin.graph();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    87
        int initialNodeCount = graph.getNodeCount();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    88
        while (!loopBegin.isDeleted()) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    89
            Mark mark = graph.getMark();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    90
            peel(loop);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    91
            canonicalizer.applyIncremental(graph, context, mark);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    92
            loop.invalidateFragments();
46344
694c102fd8ed 8177046: Update Graal
iveresov
parents: 43972
diff changeset
    93
            if (graph.getNodeCount() > initialNodeCount + MaximumDesiredSize.getValue(graph.getOptions()) * 2) {
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    94
                throw new RetryableBailoutException("FullUnroll : Graph seems to grow out of proportion");
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    95
            }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    96
        }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    97
    }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    98
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
    99
    public static void unswitch(LoopEx loop, List<ControlSplitNode> controlSplitNodeSet) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   100
        ControlSplitNode firstNode = controlSplitNodeSet.iterator().next();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   101
        LoopFragmentWhole originalLoop = loop.whole();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   102
        StructuredGraph graph = firstNode.graph();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   103
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   104
        loop.loopBegin().incrementUnswitches();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   105
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   106
        // create new control split out of loop
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   107
        ControlSplitNode newControlSplit = (ControlSplitNode) firstNode.copyWithInputs();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   108
        originalLoop.entryPoint().replaceAtPredecessor(newControlSplit);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   109
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   110
        /*
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   111
         * The code below assumes that all of the control split nodes have the same successor
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   112
         * structure, which should have been enforced by findUnswitchable.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   113
         */
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   114
        Iterator<Position> successors = firstNode.successorPositions().iterator();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   115
        assert successors.hasNext();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   116
        // original loop is used as first successor
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   117
        Position firstPosition = successors.next();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   118
        AbstractBeginNode originalLoopBegin = BeginNode.begin(originalLoop.entryPoint());
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   119
        firstPosition.set(newControlSplit, originalLoopBegin);
50330
2cbc42a5764b 8202670: Update Graal
dlong
parents: 48861
diff changeset
   120
        originalLoopBegin.setNodeSourcePosition(firstPosition.get(firstNode).getNodeSourcePosition());
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   121
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   122
        while (successors.hasNext()) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   123
            Position position = successors.next();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   124
            // create a new loop duplicate and connect it.
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   125
            LoopFragmentWhole duplicateLoop = originalLoop.duplicate();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   126
            AbstractBeginNode newBegin = BeginNode.begin(duplicateLoop.entryPoint());
50330
2cbc42a5764b 8202670: Update Graal
dlong
parents: 48861
diff changeset
   127
            newBegin.setNodeSourcePosition(position.get(firstNode).getNodeSourcePosition());
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   128
            position.set(newControlSplit, newBegin);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   129
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   130
            // For each cloned ControlSplitNode, simplify the proper path
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   131
            for (ControlSplitNode controlSplitNode : controlSplitNodeSet) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   132
                ControlSplitNode duplicatedControlSplit = duplicateLoop.getDuplicatedNode(controlSplitNode);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   133
                if (duplicatedControlSplit.isAlive()) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   134
                    AbstractBeginNode survivingSuccessor = (AbstractBeginNode) position.get(duplicatedControlSplit);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   135
                    survivingSuccessor.replaceAtUsages(InputType.Guard, newBegin);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   136
                    graph.removeSplitPropagate(duplicatedControlSplit, survivingSuccessor);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   137
                }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   138
            }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   139
        }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   140
        // original loop is simplified last to avoid deleting controlSplitNode too early
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   141
        for (ControlSplitNode controlSplitNode : controlSplitNodeSet) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   142
            if (controlSplitNode.isAlive()) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   143
                AbstractBeginNode survivingSuccessor = (AbstractBeginNode) firstPosition.get(controlSplitNode);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   144
                survivingSuccessor.replaceAtUsages(InputType.Guard, originalLoopBegin);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   145
                graph.removeSplitPropagate(controlSplitNode, survivingSuccessor);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   146
            }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   147
        }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   148
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   149
        // TODO (gd) probabilities need some amount of fixup.. (probably also in other transforms)
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   150
    }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   151
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   152
    public static void partialUnroll(LoopEx loop, EconomicMap<LoopBeginNode, OpaqueNode> opaqueUnrolledStrides) {
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   153
        assert loop.loopBegin().isMainLoop();
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   154
        loop.loopBegin().graph().getDebug().log("LoopPartialUnroll %s", loop);
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
   155
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   156
        LoopFragmentInside newSegment = loop.inside().duplicate();
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   157
        newSegment.insertWithinAfter(loop, opaqueUnrolledStrides);
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   158
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   159
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   160
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   161
    // This function splits candidate loops into pre, main and post loops,
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   162
    // dividing the iteration space to facilitate the majority of iterations
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   163
    // being executed in a main loop, which will have RCE implemented upon it.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   164
    // The initial loop form is constrained to single entry/exit, but can have
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   165
    // flow. The translation looks like:
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   166
    //
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   167
    //  @formatter:off
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   168
    //
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   169
    //       (Simple Loop entry)                   (Pre Loop Entry)
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   170
    //                |                                  |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   171
    //         (LoopBeginNode)                    (LoopBeginNode)
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   172
    //                |                                  |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   173
    //       (Loop Control Test)<------   ==>  (Loop control Test)<------
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   174
    //         /               \       \         /               \       \
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   175
    //    (Loop Exit)      (Loop Body) |    (Loop Exit)      (Loop Body) |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   176
    //        |                |       |        |                |       |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   177
    // (continue code)     (Loop End)  |  if (M < length)*   (Loop End)  |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   178
    //                         \       /       /      \           \      /
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   179
    //                          ----->        /       |            ----->
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   180
    //                                       /  if ( ... )*
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   181
    //                                      /     /       \
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   182
    //                                     /     /         \
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   183
    //                                    /     /           \
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   184
    //                                   |     /     (Main Loop Entry)
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   185
    //                                   |    |             |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   186
    //                                   |    |      (LoopBeginNode)
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   187
    //                                   |    |             |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   188
    //                                   |    |     (Loop Control Test)<------
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   189
    //                                   |    |      /               \        \
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   190
    //                                   |    |  (Loop Exit)      (Loop Body) |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   191
    //                                    \   \      |                |       |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   192
    //                                     \   \     |            (Loop End)  |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   193
    //                                      \   \    |                \       /
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   194
    //                                       \   \   |                 ------>
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   195
    //                                        \   \  |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   196
    //                                      (Main Loop Merge)*
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   197
    //                                               |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   198
    //                                      (Post Loop Entry)
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   199
    //                                               |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   200
    //                                        (LoopBeginNode)
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   201
    //                                               |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   202
    //                                       (Loop Control Test)<-----
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   203
    //                                        /               \       \
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   204
    //                                    (Loop Exit)     (Loop Body) |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   205
    //                                        |               |       |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   206
    //                                 (continue code)    (Loop End)  |
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   207
    //                                                         \      /
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   208
    //                                                          ----->
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   209
    //
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   210
    // Key: "*" = optional.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   211
    // @formatter:on
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   212
    //
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   213
    // The value "M" is the maximal value of the loop trip for the original
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   214
    // loop. The value of "length" is applicable to the number of arrays found
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   215
    // in the loop but is reduced if some or all of the arrays are known to be
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   216
    // the same length as "M". The maximum number of tests can be equal to the
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   217
    // number of arrays in the loop, where multiple instances of an array are
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   218
    // subsumed into a single test for that arrays length.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   219
    //
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   220
    // If the optional main loop entry tests are absent, the Pre Loop exit
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   221
    // connects to the Main loops entry and there is no merge hanging off the
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   222
    // main loops exit to converge flow from said tests. All split use data
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   223
    // flow is mitigated through phi(s) in the main merge if present and
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   224
    // passed through the main and post loop phi(s) from the originating pre
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   225
    // loop with final phi(s) and data flow patched to the "continue code".
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   226
    // The pre loop is constrained to one iteration for now and will likely
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   227
    // be updated to produce vector alignment if applicable.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   228
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   229
    public static LoopBeginNode insertPrePostLoops(LoopEx loop) {
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   230
        StructuredGraph graph = loop.loopBegin().graph();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   231
        graph.getDebug().log("LoopTransformations.insertPrePostLoops %s", loop);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   232
        LoopFragmentWhole preLoop = loop.whole();
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
   233
        CountedLoopInfo preCounted = loop.counted();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   234
        IfNode preLimit = preCounted.getLimitTest();
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   235
        assert preLimit != null;
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   236
        LoopBeginNode preLoopBegin = loop.loopBegin();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   237
        LoopExitNode preLoopExitNode = preLoopBegin.getSingleLoopExit();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   238
        FixedNode continuationNode = preLoopExitNode.next();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   239
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   240
        // Each duplication is inserted after the original, ergo create the post loop first
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   241
        LoopFragmentWhole mainLoop = preLoop.duplicate();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   242
        LoopFragmentWhole postLoop = preLoop.duplicate();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   243
        preLoopBegin.incrementSplits();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   244
        preLoopBegin.incrementSplits();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   245
        preLoopBegin.setPreLoop();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   246
        graph.getDebug().dump(DebugContext.VERBOSE_LEVEL, graph, "After duplication");
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   247
        LoopBeginNode mainLoopBegin = mainLoop.getDuplicatedNode(preLoopBegin);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   248
        mainLoopBegin.setMainLoop();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   249
        LoopBeginNode postLoopBegin = postLoop.getDuplicatedNode(preLoopBegin);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   250
        postLoopBegin.setPostLoop();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   251
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   252
        EndNode postEndNode = getBlockEndAfterLoopExit(postLoopBegin);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   253
        AbstractMergeNode postMergeNode = postEndNode.merge();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   254
        LoopExitNode postLoopExitNode = postLoopBegin.getSingleLoopExit();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   255
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   256
        // Update the main loop phi initialization to carry from the pre loop
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   257
        for (PhiNode prePhiNode : preLoopBegin.phis()) {
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   258
            PhiNode mainPhiNode = mainLoop.getDuplicatedNode(prePhiNode);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   259
            mainPhiNode.setValueAt(0, prePhiNode);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   260
        }
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   261
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   262
        EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopBegin);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   263
        AbstractMergeNode mainMergeNode = mainEndNode.merge();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   264
        AbstractEndNode postEntryNode = postLoopBegin.forwardEnd();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   265
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   266
        // In the case of no Bounds tests, we just flow right into the main loop
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   267
        AbstractBeginNode mainLandingNode = BeginNode.begin(postEntryNode);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   268
        LoopExitNode mainLoopExitNode = mainLoopBegin.getSingleLoopExit();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   269
        mainLoopExitNode.setNext(mainLandingNode);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   270
        preLoopExitNode.setNext(mainLoopBegin.forwardEnd());
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   271
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   272
        // Add and update any phi edges as per merge usage as needed and update usages
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   273
        processPreLoopPhis(loop, mainLoop, postLoop);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   274
        continuationNode.predecessor().clearSuccessors();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   275
        postLoopExitNode.setNext(continuationNode);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   276
        cleanupMerge(postMergeNode, postLoopExitNode);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   277
        cleanupMerge(mainMergeNode, mainLandingNode);
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   278
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   279
        // Change the preLoop to execute one iteration for now
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   280
        updatePreLoopLimit(preCounted);
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   281
        preLoopBegin.setLoopFrequency(1);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   282
        mainLoopBegin.setLoopFrequency(Math.max(0.0, mainLoopBegin.loopFrequency() - 2));
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   283
        postLoopBegin.setLoopFrequency(Math.max(0.0, postLoopBegin.loopFrequency() - 1));
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
   284
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   285
        // The pre and post loops don't require safepoints at all
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   286
        for (SafepointNode safepoint : preLoop.nodes().filter(SafepointNode.class)) {
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   287
            graph.removeFixed(safepoint);
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   288
        }
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   289
        for (SafepointNode safepoint : postLoop.nodes().filter(SafepointNode.class)) {
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   290
            graph.removeFixed(safepoint);
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   291
        }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   292
        graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "InsertPrePostLoops %s", loop);
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   293
        return mainLoopBegin;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   294
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   295
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   296
    /**
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   297
     * Cleanup the merge and remove the predecessors too.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   298
     */
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   299
    private static void cleanupMerge(AbstractMergeNode mergeNode, AbstractBeginNode landingNode) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   300
        for (EndNode end : mergeNode.cfgPredecessors().snapshot()) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   301
            mergeNode.removeEnd(end);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   302
            end.safeDelete();
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   303
        }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   304
        mergeNode.prepareDelete(landingNode);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   305
        mergeNode.safeDelete();
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   306
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   307
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   308
    private static void processPreLoopPhis(LoopEx preLoop, LoopFragmentWhole mainLoop, LoopFragmentWhole postLoop) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   309
        // process phis for the post loop
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   310
        LoopBeginNode preLoopBegin = preLoop.loopBegin();
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   311
        for (PhiNode prePhiNode : preLoopBegin.phis()) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   312
            PhiNode postPhiNode = postLoop.getDuplicatedNode(prePhiNode);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   313
            PhiNode mainPhiNode = mainLoop.getDuplicatedNode(prePhiNode);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   314
            postPhiNode.setValueAt(0, mainPhiNode);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   315
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   316
            // Build a work list to update the pre loop phis to the post loops phis
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   317
            for (Node usage : prePhiNode.usages().snapshot()) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   318
                if (usage == mainPhiNode) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   319
                    continue;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   320
                }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   321
                if (preLoop.isOutsideLoop(usage)) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   322
                    usage.replaceFirstInput(prePhiNode, postPhiNode);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   323
                }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   324
            }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   325
        }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   326
        for (Node node : preLoop.inside().nodes()) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   327
            for (Node externalUsage : node.usages().snapshot()) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   328
                if (preLoop.isOutsideLoop(externalUsage)) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   329
                    Node postUsage = postLoop.getDuplicatedNode(node);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   330
                    assert postUsage != null;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   331
                    externalUsage.replaceFirstInput(node, postUsage);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   332
                }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   333
            }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   334
        }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   335
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   336
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   337
    /**
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   338
     * Find the end of the block following the LoopExit.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   339
     */
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   340
    private static EndNode getBlockEndAfterLoopExit(LoopBeginNode curLoopBegin) {
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
   341
        FixedNode node = curLoopBegin.getSingleLoopExit().next();
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   342
        // Find the last node after the exit blocks starts
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   343
        return getBlockEnd(node);
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   344
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   345
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   346
    private static EndNode getBlockEnd(FixedNode node) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   347
        FixedNode curNode = node;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   348
        while (curNode instanceof FixedWithNextNode) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   349
            curNode = ((FixedWithNextNode) curNode).next();
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   350
        }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   351
        return (EndNode) curNode;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   352
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   353
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   354
    private static void updatePreLoopLimit(CountedLoopInfo preCounted) {
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   355
        // Update the pre loops limit test
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   356
        // Make new limit one iteration
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   357
        ValueNode newLimit = AddNode.add(preCounted.getStart(), preCounted.getCounter().strideNode(), NodeView.DEFAULT);
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   358
        // Fetch the variable we are not replacing and configure the one we are
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   359
        ValueNode ub = preCounted.getLimit();
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   360
        LogicNode entryCheck;
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   361
        if (preCounted.getDirection() == Direction.Up) {
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   362
            entryCheck = IntegerLessThanNode.create(newLimit, ub, NodeView.DEFAULT);
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   363
        } else {
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   364
            entryCheck = IntegerLessThanNode.create(ub, newLimit, NodeView.DEFAULT);
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   365
        }
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   366
        newLimit = ConditionalNode.create(entryCheck, newLimit, ub, NodeView.DEFAULT);
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   367
        // Re-wire the condition with the new limit
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   368
        CompareNode compareNode = (CompareNode) preCounted.getLimitTest().condition();
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   369
        compareNode.replaceFirstInput(ub, compareNode.graph().addOrUniqueWithInputs(newLimit));
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   370
    }
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   371
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   372
    public static List<ControlSplitNode> findUnswitchable(LoopEx loop) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   373
        List<ControlSplitNode> controls = null;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   374
        ValueNode invariantValue = null;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   375
        for (IfNode ifNode : loop.whole().nodes().filter(IfNode.class)) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   376
            if (loop.isOutsideLoop(ifNode.condition())) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   377
                if (controls == null) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   378
                    invariantValue = ifNode.condition();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   379
                    controls = new ArrayList<>();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   380
                    controls.add(ifNode);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   381
                } else if (ifNode.condition() == invariantValue) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   382
                    controls.add(ifNode);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   383
                }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   384
            }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   385
        }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   386
        if (controls == null) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   387
            SwitchNode firstSwitch = null;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   388
            for (SwitchNode switchNode : loop.whole().nodes().filter(SwitchNode.class)) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   389
                if (switchNode.successors().count() > 1 && loop.isOutsideLoop(switchNode.value())) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   390
                    if (controls == null) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   391
                        firstSwitch = switchNode;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   392
                        invariantValue = switchNode.value();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   393
                        controls = new ArrayList<>();
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   394
                        controls.add(switchNode);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   395
                    } else if (switchNode.value() == invariantValue && firstSwitch.structureEquals(switchNode)) {
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   396
                        // Only collect switches which test the same values in the same order
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   397
                        controls.add(switchNode);
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   398
                    }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   399
                }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   400
            }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   401
        }
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   402
        return controls;
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   403
    }
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   404
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   405
    public static boolean isUnrollableLoop(LoopEx loop) {
46807
8b2c620d7092 8186158: Update Graal
iveresov
parents: 46762
diff changeset
   406
        if (!loop.isCounted() || !loop.counted().getCounter().isConstantStride() || !loop.loop().getChildren().isEmpty()) {
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   407
            return false;
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   408
        }
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   409
        assert loop.counted().getDirection() != null;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   410
        LoopBeginNode loopBegin = loop.loopBegin();
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   411
        LogicNode condition = loop.counted().getLimitTest().condition();
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   412
        if (!(condition instanceof CompareNode)) {
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   413
            return false;
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   414
        }
48861
47f19ff9903c 8194819: Update Graal
iveresov
parents: 47216
diff changeset
   415
        if (((CompareNode) condition).condition() == CanonicalCondition.EQ) {
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   416
            condition.getDebug().log(DebugContext.VERBOSE_LEVEL, "isUnrollableLoop %s condition unsupported %s ", loopBegin, ((CompareNode) condition).condition());
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   417
            return false;
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   418
        }
52578
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   419
        long stride = loop.counted().getCounter().constantStride();
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   420
        try {
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   421
            Math.addExact(stride, stride);
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   422
        } catch (ArithmeticException ae) {
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   423
            condition.getDebug().log(DebugContext.VERBOSE_LEVEL, "isUnrollableLoop %s doubling the stride overflows %d", loopBegin, stride);
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   424
            return false;
7dd81e82d083 8210777: Update Graal
dlong
parents: 50858
diff changeset
   425
        }
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   426
        if (loopBegin.isMainLoop() || loopBegin.isSimpleLoop()) {
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   427
            // Flow-less loops to partial unroll for now. 3 blocks corresponds to an if that either
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   428
            // exits or continues the loop. There might be fixed and floating work within the loop
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   429
            // as well.
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   430
            if (loop.loop().getBlocks().size() < 3) {
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
   431
                return true;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   432
            }
46963
089674d9949b 8186681: Update Graal
iveresov
parents: 46807
diff changeset
   433
            condition.getDebug().log(DebugContext.VERBOSE_LEVEL, "isUnrollableLoop %s too large to unroll %s ", loopBegin, loop.loop().getBlocks().size());
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   434
        }
46762
f7defa99f173 8185829: Update Graal
dlong
parents: 46640
diff changeset
   435
        return false;
46640
70bdce04c59b 8183991: Update Graal
iveresov
parents: 46344
diff changeset
   436
    }
43972
1ade39b8381b 8174879: Rename jdk.vm.ci to jdk.internal.vm.ci
kvn
parents:
diff changeset
   437
}