hotspot/src/share/vm/opto/loopTransform.cpp
author kvn
Sat, 02 Apr 2011 10:54:15 -0700
changeset 9101 ff58f9a8e31c
parent 8884 9e2b81253ef9
child 9104 85edf8ec98b2
permissions -rw-r--r--
7004535: Clone loop predicate during loop unswitch Summary: Clone loop predicate for clonned loops Reviewed-by: never
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     1
/*
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
     2
 * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     4
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
489c9b5090e2 Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
489c9b5090e2 Initial load
duke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     8
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
489c9b5090e2 Initial load
duke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
489c9b5090e2 Initial load
duke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
489c9b5090e2 Initial load
duke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
489c9b5090e2 Initial load
duke
parents:
diff changeset
    13
 * accompanied this code).
489c9b5090e2 Initial load
duke
parents:
diff changeset
    14
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
489c9b5090e2 Initial load
duke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
489c9b5090e2 Initial load
duke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    18
 *
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    21
 * questions.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    22
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    23
 */
489c9b5090e2 Initial load
duke
parents:
diff changeset
    24
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    25
#include "precompiled.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    26
#include "compiler/compileLog.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    27
#include "memory/allocation.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    28
#include "opto/addnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    29
#include "opto/callnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    30
#include "opto/connode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    31
#include "opto/divnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    32
#include "opto/loopnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    33
#include "opto/mulnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    34
#include "opto/rootnode.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    35
#include "opto/runtime.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6770
diff changeset
    36
#include "opto/subnode.hpp"
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    37
489c9b5090e2 Initial load
duke
parents:
diff changeset
    38
//------------------------------is_loop_exit-----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    39
// Given an IfNode, return the loop-exiting projection or NULL if both
489c9b5090e2 Initial load
duke
parents:
diff changeset
    40
// arms remain in the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    41
Node *IdealLoopTree::is_loop_exit(Node *iff) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    42
  if( iff->outcnt() != 2 ) return NULL; // Ignore partially dead tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
    43
  PhaseIdealLoop *phase = _phase;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    44
  // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
    45
  // we need loop unswitching instead of peeling.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    46
  if( !is_member(phase->get_loop( iff->raw_out(0) )) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
    47
    return iff->raw_out(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    48
  if( !is_member(phase->get_loop( iff->raw_out(1) )) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
    49
    return iff->raw_out(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    50
  return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    51
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    52
489c9b5090e2 Initial load
duke
parents:
diff changeset
    53
489c9b5090e2 Initial load
duke
parents:
diff changeset
    54
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
    55
489c9b5090e2 Initial load
duke
parents:
diff changeset
    56
489c9b5090e2 Initial load
duke
parents:
diff changeset
    57
//------------------------------record_for_igvn----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    58
// Put loop body on igvn work list
489c9b5090e2 Initial load
duke
parents:
diff changeset
    59
void IdealLoopTree::record_for_igvn() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    60
  for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    61
    Node *n = _body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    62
    _phase->_igvn._worklist.push(n);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    63
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    64
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    65
489c9b5090e2 Initial load
duke
parents:
diff changeset
    66
//------------------------------compute_profile_trip_cnt----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    67
// Compute loop trip count from profile data as
489c9b5090e2 Initial load
duke
parents:
diff changeset
    68
//    (backedge_count + loop_exit_count) / loop_exit_count
489c9b5090e2 Initial load
duke
parents:
diff changeset
    69
void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    70
  if (!_head->is_CountedLoop()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    71
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    72
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    73
  CountedLoopNode* head = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    74
  if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    75
    return; // Already computed
489c9b5090e2 Initial load
duke
parents:
diff changeset
    76
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    77
  float trip_cnt = (float)max_jint; // default is big
489c9b5090e2 Initial load
duke
parents:
diff changeset
    78
489c9b5090e2 Initial load
duke
parents:
diff changeset
    79
  Node* back = head->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    80
  while (back != head) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    81
    if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    82
        back->in(0) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    83
        back->in(0)->is_If() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    84
        back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    85
        back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    86
      break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    87
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    88
    back = phase->idom(back);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    89
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    90
  if (back != head) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    91
    assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    92
           back->in(0), "if-projection exists");
489c9b5090e2 Initial load
duke
parents:
diff changeset
    93
    IfNode* back_if = back->in(0)->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    94
    float loop_back_cnt = back_if->_fcnt * back_if->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    95
489c9b5090e2 Initial load
duke
parents:
diff changeset
    96
    // Now compute a loop exit count
489c9b5090e2 Initial load
duke
parents:
diff changeset
    97
    float loop_exit_cnt = 0.0f;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    98
    for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    99
      Node *n = _body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   100
      if( n->is_If() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   101
        IfNode *iff = n->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   102
        if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   103
          Node *exit = is_loop_exit(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   104
          if( exit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   105
            float exit_prob = iff->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   106
            if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   107
            if (exit_prob > PROB_MIN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   108
              float exit_cnt = iff->_fcnt * exit_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   109
              loop_exit_cnt += exit_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   110
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   111
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   112
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   113
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   114
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   115
    if (loop_exit_cnt > 0.0f) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   116
      trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   117
    } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   118
      // No exit count so use
489c9b5090e2 Initial load
duke
parents:
diff changeset
   119
      trip_cnt = loop_back_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   120
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   121
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   122
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   123
  if (TraceProfileTripCount) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   124
    tty->print_cr("compute_profile_trip_cnt  lp: %d cnt: %f\n", head->_idx, trip_cnt);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   125
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   126
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   127
  head->set_profile_trip_cnt(trip_cnt);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   128
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   129
489c9b5090e2 Initial load
duke
parents:
diff changeset
   130
//---------------------is_invariant_addition-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   131
// Return nonzero index of invariant operand for an Add or Sub
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1500
diff changeset
   132
// of (nonconstant) invariant and variant values. Helper for reassociate_invariants.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   133
int IdealLoopTree::is_invariant_addition(Node* n, PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   134
  int op = n->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   135
  if (op == Op_AddI || op == Op_SubI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   136
    bool in1_invar = this->is_invariant(n->in(1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   137
    bool in2_invar = this->is_invariant(n->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   138
    if (in1_invar && !in2_invar) return 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   139
    if (!in1_invar && in2_invar) return 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   140
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   141
  return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   142
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   143
489c9b5090e2 Initial load
duke
parents:
diff changeset
   144
//---------------------reassociate_add_sub-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   145
// Reassociate invariant add and subtract expressions:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   146
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
   147
// inv1 + (x + inv2)  =>  ( inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   148
// (x + inv2) + inv1  =>  ( inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   149
// inv1 + (x - inv2)  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   150
// inv1 - (inv2 - x)  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   151
// (x + inv2) - inv1  =>  (-inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   152
// (x - inv2) + inv1  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   153
// (x - inv2) - inv1  =>  (-inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   154
// inv1 + (inv2 - x)  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   155
// inv1 - (x - inv2)  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   156
// (inv2 - x) + inv1  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   157
// (inv2 - x) - inv1  =>  (-inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   158
// inv1 - (x + inv2)  =>  ( inv1 - inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   159
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
   160
Node* IdealLoopTree::reassociate_add_sub(Node* n1, PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   161
  if (!n1->is_Add() && !n1->is_Sub() || n1->outcnt() == 0) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   162
  if (is_invariant(n1)) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   163
  int inv1_idx = is_invariant_addition(n1, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   164
  if (!inv1_idx) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   165
  // Don't mess with add of constant (igvn moves them to expression tree root.)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   166
  if (n1->is_Add() && n1->in(2)->is_Con()) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   167
  Node* inv1 = n1->in(inv1_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   168
  Node* n2 = n1->in(3 - inv1_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   169
  int inv2_idx = is_invariant_addition(n2, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   170
  if (!inv2_idx) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   171
  Node* x    = n2->in(3 - inv2_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   172
  Node* inv2 = n2->in(inv2_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   173
489c9b5090e2 Initial load
duke
parents:
diff changeset
   174
  bool neg_x    = n2->is_Sub() && inv2_idx == 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   175
  bool neg_inv2 = n2->is_Sub() && inv2_idx == 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   176
  bool neg_inv1 = n1->is_Sub() && inv1_idx == 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   177
  if (n1->is_Sub() && inv1_idx == 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   178
    neg_x    = !neg_x;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   179
    neg_inv2 = !neg_inv2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   180
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   181
  Node* inv1_c = phase->get_ctrl(inv1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   182
  Node* inv2_c = phase->get_ctrl(inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   183
  Node* n_inv1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   184
  if (neg_inv1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   185
    Node *zero = phase->_igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   186
    phase->set_ctrl(zero, phase->C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   187
    n_inv1 = new (phase->C, 3) SubINode(zero, inv1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   188
    phase->register_new_node(n_inv1, inv1_c);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   189
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   190
    n_inv1 = inv1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   191
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   192
  Node* inv;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   193
  if (neg_inv2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   194
    inv = new (phase->C, 3) SubINode(n_inv1, inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   195
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   196
    inv = new (phase->C, 3) AddINode(n_inv1, inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   197
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   198
  phase->register_new_node(inv, phase->get_early_ctrl(inv));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   199
489c9b5090e2 Initial load
duke
parents:
diff changeset
   200
  Node* addx;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   201
  if (neg_x) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   202
    addx = new (phase->C, 3) SubINode(inv, x);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   203
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   204
    addx = new (phase->C, 3) AddINode(x, inv);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   205
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   206
  phase->register_new_node(addx, phase->get_ctrl(x));
5901
c046f8e9c52b 6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents: 5547
diff changeset
   207
  phase->_igvn.replace_node(n1, addx);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   208
  assert(phase->get_loop(phase->get_ctrl(n1)) == this, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   209
  _body.yank(n1);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   210
  return addx;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   211
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   212
489c9b5090e2 Initial load
duke
parents:
diff changeset
   213
//---------------------reassociate_invariants-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   214
// Reassociate invariant expressions:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   215
void IdealLoopTree::reassociate_invariants(PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   216
  for (int i = _body.size() - 1; i >= 0; i--) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   217
    Node *n = _body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   218
    for (int j = 0; j < 5; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   219
      Node* nn = reassociate_add_sub(n, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   220
      if (nn == NULL) break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   221
      n = nn; // again
489c9b5090e2 Initial load
duke
parents:
diff changeset
   222
    };
489c9b5090e2 Initial load
duke
parents:
diff changeset
   223
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   224
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   225
489c9b5090e2 Initial load
duke
parents:
diff changeset
   226
//------------------------------policy_peeling---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   227
// Return TRUE or FALSE if the loop should be peeled or not.  Peel if we can
489c9b5090e2 Initial load
duke
parents:
diff changeset
   228
// make some loop-invariant test (usually a null-check) happen before the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   229
bool IdealLoopTree::policy_peeling( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   230
  Node *test = ((IdealLoopTree*)this)->tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   231
  int  body_size = ((IdealLoopTree*)this)->_body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   232
  int  uniq      = phase->C->unique();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   233
  // Peeling does loop cloning which can result in O(N^2) node construction
489c9b5090e2 Initial load
duke
parents:
diff changeset
   234
  if( body_size > 255 /* Prevent overflow for large body_size */
489c9b5090e2 Initial load
duke
parents:
diff changeset
   235
      || (body_size * body_size + uniq > MaxNodeLimit) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   236
    return false;           // too large to safely clone
489c9b5090e2 Initial load
duke
parents:
diff changeset
   237
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   238
  while( test != _head ) {      // Scan till run off top of loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   239
    if( test->is_If() ) {       // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   240
      Node *ctrl = phase->get_ctrl(test->in(1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   241
      if (ctrl->is_top())
489c9b5090e2 Initial load
duke
parents:
diff changeset
   242
        return false;           // Found dead test on live IF?  No peeling!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   243
      // Standard IF only has one input value to check for loop invariance
489c9b5090e2 Initial load
duke
parents:
diff changeset
   244
      assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   245
      // Condition is not a member of this loop?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   246
      if( !is_member(phase->get_loop(ctrl)) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   247
          is_loop_exit(test) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   248
        return true;            // Found reason to peel!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   249
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   250
    // Walk up dominators to loop _head looking for test which is
489c9b5090e2 Initial load
duke
parents:
diff changeset
   251
    // executed on every path thru loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   252
    test = phase->idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   253
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   254
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   255
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   256
489c9b5090e2 Initial load
duke
parents:
diff changeset
   257
//------------------------------peeled_dom_test_elim---------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   258
// If we got the effect of peeling, either by actually peeling or by making
489c9b5090e2 Initial load
duke
parents:
diff changeset
   259
// a pre-loop which must execute at least once, we can remove all
489c9b5090e2 Initial load
duke
parents:
diff changeset
   260
// loop-invariant dominated tests in the main body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   261
void PhaseIdealLoop::peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   262
  bool progress = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   263
  while( progress ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   264
    progress = false;           // Reset for next iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
   265
    Node *prev = loop->_head->in(LoopNode::LoopBackControl);//loop->tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   266
    Node *test = prev->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   267
    while( test != loop->_head ) { // Scan till run off top of loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   268
489c9b5090e2 Initial load
duke
parents:
diff changeset
   269
      int p_op = prev->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   270
      if( (p_op == Op_IfFalse || p_op == Op_IfTrue) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   271
          test->is_If() &&      // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   272
          !test->in(1)->is_Con() && // And not already obvious?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   273
          // Condition is not a member of this loop?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   274
          !loop->is_member(get_loop(get_ctrl(test->in(1))))){
489c9b5090e2 Initial load
duke
parents:
diff changeset
   275
        // Walk loop body looking for instances of this test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   276
        for( uint i = 0; i < loop->_body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   277
          Node *n = loop->_body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   278
          if( n->is_If() && n->in(1) == test->in(1) /*&& n != loop->tail()->in(0)*/ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   279
            // IfNode was dominated by version in peeled loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
   280
            progress = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   281
            dominated_by( old_new[prev->_idx], n );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   282
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   283
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   284
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   285
      prev = test;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   286
      test = idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   287
    } // End of scan tests in loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   288
489c9b5090e2 Initial load
duke
parents:
diff changeset
   289
  } // End of while( progress )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   290
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   291
489c9b5090e2 Initial load
duke
parents:
diff changeset
   292
//------------------------------do_peeling-------------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   293
// Peel the first iteration of the given loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   294
// Step 1: Clone the loop body.  The clone becomes the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   295
//         The pre-loop illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   296
// Step 2: Make the old-loop fall-in edges point to the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   297
//         Do this by making the old-loop fall-in edges act as if they came
489c9b5090e2 Initial load
duke
parents:
diff changeset
   298
//         around the loopback from the prior iteration (follow the old-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   299
//         backedges) and then map to the new peeled iteration.  This leaves
489c9b5090e2 Initial load
duke
parents:
diff changeset
   300
//         the pre-loop with only 1 user (the new peeled iteration), but the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   301
//         peeled-loop backedge has 2 users.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   302
// Step 3: Cut the backedge on the clone (so its not a loop) and remove the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   303
//         extra backedge user.
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   304
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   305
//                   orig
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   306
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   307
//                  stmt1
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   308
//                    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   309
//                    v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   310
//              loop predicate
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   311
//                    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   312
//                    v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   313
//                   loop<----+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   314
//                     |      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   315
//                   stmt2    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   316
//                     |      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   317
//                     v      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   318
//                    if      ^
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   319
//                   / \      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   320
//                  /   \     |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   321
//                 v     v    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   322
//               false true   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   323
//               /       \    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   324
//              /         ----+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   325
//             |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   326
//             v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   327
//           exit
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   328
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   329
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   330
//            after clone loop
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   331
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   332
//                   stmt1
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   333
//                     |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   334
//                     v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   335
//               loop predicate
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   336
//                 /       \
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   337
//        clone   /         \   orig
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   338
//               /           \
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   339
//              /             \
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   340
//             v               v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   341
//   +---->loop clone          loop<----+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   342
//   |      |                    |      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   343
//   |    stmt2 clone          stmt2    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   344
//   |      |                    |      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   345
//   |      v                    v      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   346
//   ^      if clone            If      ^
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   347
//   |      / \                / \      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   348
//   |     /   \              /   \     |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   349
//   |    v     v            v     v    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   350
//   |    true  false      false true   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   351
//   |    /         \      /       \    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   352
//   +----           \    /         ----+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   353
//                    \  /
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   354
//                    1v v2
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   355
//                  region
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   356
//                     |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   357
//                     v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   358
//                   exit
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   359
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   360
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   361
//         after peel and predicate move
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   362
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   363
//                   stmt1
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   364
//                    /
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   365
//                   /
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   366
//        clone     /            orig
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   367
//                 /
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   368
//                /              +----------+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   369
//               /               |          |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   370
//              /          loop predicate   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   371
//             /                 |          |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   372
//            v                  v          |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   373
//   TOP-->loop clone          loop<----+   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   374
//          |                    |      |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   375
//        stmt2 clone          stmt2    |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   376
//          |                    |      |   ^
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   377
//          v                    v      |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   378
//          if clone            If      ^   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   379
//          / \                / \      |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   380
//         /   \              /   \     |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   381
//        v     v            v     v    |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   382
//      true   false      false  true   |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   383
//        |         \      /       \    |   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   384
//        |          \    /         ----+   ^
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   385
//        |           \  /                  |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   386
//        |           1v v2                 |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   387
//        v         region                  |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   388
//        |            |                    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   389
//        |            v                    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   390
//        |          exit                   |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   391
//        |                                 |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   392
//        +--------------->-----------------+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   393
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   394
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   395
//              final graph
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   396
//
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   397
//                  stmt1
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   398
//                    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   399
//                    v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   400
//                  stmt2 clone
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   401
//                    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   402
//                    v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   403
//                   if clone
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   404
//                  / |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   405
//                 /  |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   406
//                v   v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   407
//            false  true
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   408
//             |      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   409
//             |      v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   410
//             | loop predicate
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   411
//             |      |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   412
//             |      v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   413
//             |     loop<----+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   414
//             |      |       |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   415
//             |    stmt2     |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   416
//             |      |       |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   417
//             |      v       |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   418
//             v      if      ^
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   419
//             |     /  \     |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   420
//             |    /    \    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   421
//             |   v     v    |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   422
//             | false  true  |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   423
//             |  |        \  |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   424
//             v  v         --+
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   425
//            region
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   426
//              |
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   427
//              v
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   428
//             exit
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   429
//
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   430
void PhaseIdealLoop::do_peeling( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   431
489c9b5090e2 Initial load
duke
parents:
diff changeset
   432
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   433
  // Peeling a 'main' loop in a pre/main/post situation obfuscates the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   434
  // 'pre' loop from the main and the 'pre' can no longer have it's
489c9b5090e2 Initial load
duke
parents:
diff changeset
   435
  // iterations adjusted.  Therefore, we need to declare this loop as
489c9b5090e2 Initial load
duke
parents:
diff changeset
   436
  // no longer a 'main' loop; it will need new pre and post loops before
489c9b5090e2 Initial load
duke
parents:
diff changeset
   437
  // we can do further RCE.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   438
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   439
  if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   440
    tty->print("Peel         ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   441
    loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   442
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   443
#endif
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   444
  Node* head = loop->_head;
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   445
  bool counted_loop = head->is_CountedLoop();
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   446
  if (counted_loop) {
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   447
    CountedLoopNode *cl = head->as_CountedLoop();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   448
    assert(cl->trip_count() > 0, "peeling a fully unrolled loop");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   449
    cl->set_trip_count(cl->trip_count() - 1);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   450
    if (cl->is_main_loop()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   451
      cl->set_normal_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   452
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   453
      if (PrintOpto && VerifyLoopOptimizations) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   454
        tty->print("Peeling a 'main' loop; resetting to 'normal' ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   455
        loop->dump_head();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   456
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   457
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   458
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   459
  }
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   460
  Node* entry = head->in(LoopNode::EntryControl);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   461
489c9b5090e2 Initial load
duke
parents:
diff changeset
   462
  // Step 1: Clone the loop body.  The clone becomes the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   463
  //         The pre-loop illegally has 2 control users (old & new loops).
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   464
  clone_loop( loop, old_new, dom_depth(head) );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   465
489c9b5090e2 Initial load
duke
parents:
diff changeset
   466
  // Step 2: Make the old-loop fall-in edges point to the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   467
  //         Do this by making the old-loop fall-in edges act as if they came
489c9b5090e2 Initial load
duke
parents:
diff changeset
   468
  //         around the loopback from the prior iteration (follow the old-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   469
  //         backedges) and then map to the new peeled iteration.  This leaves
489c9b5090e2 Initial load
duke
parents:
diff changeset
   470
  //         the pre-loop with only 1 user (the new peeled iteration), but the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   471
  //         peeled-loop backedge has 2 users.
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   472
  Node* new_exit_value = old_new[head->in(LoopNode::LoopBackControl)->_idx];
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   473
  new_exit_value = move_loop_predicates(entry, new_exit_value);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   474
  _igvn.hash_delete(head);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   475
  head->set_req(LoopNode::EntryControl, new_exit_value);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   476
  for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) {
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   477
    Node* old = head->fast_out(j);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   478
    if (old->in(0) == loop->_head && old->req() == 3 && old->is_Phi()) {
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   479
      new_exit_value = old_new[old->in(LoopNode::LoopBackControl)->_idx];
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   480
      if (!new_exit_value )     // Backedge value is ALSO loop invariant?
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   481
        // Then loop body backedge value remains the same.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   482
        new_exit_value = old->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   483
      _igvn.hash_delete(old);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   484
      old->set_req(LoopNode::EntryControl, new_exit_value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   485
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   486
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   487
489c9b5090e2 Initial load
duke
parents:
diff changeset
   488
489c9b5090e2 Initial load
duke
parents:
diff changeset
   489
  // Step 3: Cut the backedge on the clone (so its not a loop) and remove the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   490
  //         extra backedge user.
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   491
  Node* new_head = old_new[head->_idx];
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   492
  _igvn.hash_delete(new_head);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   493
  new_head->set_req(LoopNode::LoopBackControl, C->top());
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   494
  for (DUIterator_Fast j2max, j2 = new_head->fast_outs(j2max); j2 < j2max; j2++) {
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   495
    Node* use = new_head->fast_out(j2);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   496
    if (use->in(0) == new_head && use->req() == 3 && use->is_Phi()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   497
      _igvn.hash_delete(use);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   498
      use->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   499
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   500
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   501
489c9b5090e2 Initial load
duke
parents:
diff changeset
   502
489c9b5090e2 Initial load
duke
parents:
diff changeset
   503
  // Step 4: Correct dom-depth info.  Set to loop-head depth.
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   504
  int dd = dom_depth(head);
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   505
  set_idom(head, head->in(1), dd);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   506
  for (uint j3 = 0; j3 < loop->_body.size(); j3++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   507
    Node *old = loop->_body.at(j3);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   508
    Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   509
    if (!has_ctrl(nnn))
489c9b5090e2 Initial load
duke
parents:
diff changeset
   510
      set_idom(nnn, idom(nnn), dd-1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   511
    // While we're at it, remove any SafePoints from the peeled code
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
   512
    if (old->Opcode() == Op_SafePoint) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   513
      Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   514
      lazy_replace(nnn,nnn->in(TypeFunc::Control));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   515
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   516
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   517
489c9b5090e2 Initial load
duke
parents:
diff changeset
   518
  // Now force out all loop-invariant dominating tests.  The optimizer
489c9b5090e2 Initial load
duke
parents:
diff changeset
   519
  // finds some, but we _know_ they are all useless.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   520
  peeled_dom_test_elim(loop,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   521
489c9b5090e2 Initial load
duke
parents:
diff changeset
   522
  loop->record_for_igvn();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   523
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   524
489c9b5090e2 Initial load
duke
parents:
diff changeset
   525
//------------------------------policy_maximally_unroll------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   526
// Return exact loop trip count, or 0 if not maximally unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   527
bool IdealLoopTree::policy_maximally_unroll( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   528
  CountedLoopNode *cl = _head->as_CountedLoop();
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   529
  assert(cl->is_normal_loop(), "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   530
489c9b5090e2 Initial load
duke
parents:
diff changeset
   531
  Node *init_n = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   532
  Node *limit_n = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   533
489c9b5090e2 Initial load
duke
parents:
diff changeset
   534
  // Non-constant bounds
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   535
  if (init_n   == NULL || !init_n->is_Con()  ||
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   536
      limit_n  == NULL || !limit_n->is_Con() ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   537
      // protect against stride not being a constant
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   538
      !cl->stride_is_con()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   539
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   540
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   541
  int init   = init_n->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   542
  int limit  = limit_n->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   543
  int span   = limit - init;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   544
  int stride = cl->stride_con();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   545
489c9b5090e2 Initial load
duke
parents:
diff changeset
   546
  if (init >= limit || stride > span) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   547
    // return a false (no maximally unroll) and the regular unroll/peel
489c9b5090e2 Initial load
duke
parents:
diff changeset
   548
    // route will make a small mess which CCP will fold away.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   549
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   550
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   551
  uint trip_count = span/stride;   // trip_count can be greater than 2 Gig.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   552
  assert( (int)trip_count*stride == span, "must divide evenly" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   553
489c9b5090e2 Initial load
duke
parents:
diff changeset
   554
  // Real policy: if we maximally unroll, does it get too big?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   555
  // Allow the unrolled mess to get larger than standard loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   556
  // size.  After all, it will no longer be a loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   557
  uint body_size    = _body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   558
  uint unroll_limit = (uint)LoopUnrollLimit * 4;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   559
  assert( (intx)unroll_limit == LoopUnrollLimit * 4, "LoopUnrollLimit must fit in 32bits");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   560
  cl->set_trip_count(trip_count);
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   561
  if (trip_count > unroll_limit || body_size > unroll_limit) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   562
    return false;
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   563
  }
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   564
8884
9e2b81253ef9 7032696: Fix for 7029152 broke VM
kvn
parents: 8879
diff changeset
   565
  // Currently we don't have policy to optimize one iteration loops.
9e2b81253ef9 7032696: Fix for 7029152 broke VM
kvn
parents: 8879
diff changeset
   566
  // Maximally unrolling transformation is used for that:
9e2b81253ef9 7032696: Fix for 7029152 broke VM
kvn
parents: 8879
diff changeset
   567
  // it is peeled and the original loop become non reachable (dead).
9e2b81253ef9 7032696: Fix for 7029152 broke VM
kvn
parents: 8879
diff changeset
   568
  if (trip_count == 1)
9e2b81253ef9 7032696: Fix for 7029152 broke VM
kvn
parents: 8879
diff changeset
   569
    return true;
9e2b81253ef9 7032696: Fix for 7029152 broke VM
kvn
parents: 8879
diff changeset
   570
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   571
  // Do not unroll a loop with String intrinsics code.
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   572
  // String intrinsics are large and have loops.
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   573
  for (uint k = 0; k < _body.size(); k++) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   574
    Node* n = _body.at(k);
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   575
    switch (n->Opcode()) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   576
      case Op_StrComp:
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   577
      case Op_StrEquals:
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   578
      case Op_StrIndexOf:
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   579
      case Op_AryEq: {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   580
        return false;
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   581
      }
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   582
    } // switch
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   583
  }
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   584
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   585
  if (body_size <= unroll_limit) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   586
    uint new_body_size = body_size * trip_count;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   587
    if (new_body_size <= unroll_limit &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   588
        body_size == new_body_size / trip_count &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   589
        // Unrolling can result in a large amount of node construction
489c9b5090e2 Initial load
duke
parents:
diff changeset
   590
        new_body_size < MaxNodeLimit - phase->C->unique()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   591
      return true;    // maximally unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   592
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   593
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   594
489c9b5090e2 Initial load
duke
parents:
diff changeset
   595
  return false;               // Do not maximally unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   596
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   597
489c9b5090e2 Initial load
duke
parents:
diff changeset
   598
489c9b5090e2 Initial load
duke
parents:
diff changeset
   599
//------------------------------policy_unroll----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   600
// Return TRUE or FALSE if the loop should be unrolled or not.  Unroll if
489c9b5090e2 Initial load
duke
parents:
diff changeset
   601
// the loop is a CountedLoop and the body is small enough.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   602
bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   603
489c9b5090e2 Initial load
duke
parents:
diff changeset
   604
  CountedLoopNode *cl = _head->as_CountedLoop();
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   605
  assert(cl->is_normal_loop() || cl->is_main_loop(), "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   606
489c9b5090e2 Initial load
duke
parents:
diff changeset
   607
  // protect against stride not being a constant
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   608
  if (!cl->stride_is_con()) return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   609
489c9b5090e2 Initial load
duke
parents:
diff changeset
   610
  // protect against over-unrolling
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   611
  if (cl->trip_count() <= 1) return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   612
489c9b5090e2 Initial load
duke
parents:
diff changeset
   613
  int future_unroll_ct = cl->unrolled_count() * 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   614
489c9b5090e2 Initial load
duke
parents:
diff changeset
   615
  // Don't unroll if the next round of unrolling would push us
489c9b5090e2 Initial load
duke
parents:
diff changeset
   616
  // over the expected trip count of the loop.  One is subtracted
489c9b5090e2 Initial load
duke
parents:
diff changeset
   617
  // from the expected trip count because the pre-loop normally
489c9b5090e2 Initial load
duke
parents:
diff changeset
   618
  // executes 1 iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   619
  if (UnrollLimitForProfileCheck > 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   620
      cl->profile_trip_cnt() != COUNT_UNKNOWN &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   621
      future_unroll_ct        > UnrollLimitForProfileCheck &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   622
      (float)future_unroll_ct > cl->profile_trip_cnt() - 1.0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   623
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   624
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   625
489c9b5090e2 Initial load
duke
parents:
diff changeset
   626
  // When unroll count is greater than LoopUnrollMin, don't unroll if:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   627
  //   the residual iterations are more than 10% of the trip count
489c9b5090e2 Initial load
duke
parents:
diff changeset
   628
  //   and rounds of "unroll,optimize" are not making significant progress
489c9b5090e2 Initial load
duke
parents:
diff changeset
   629
  //   Progress defined as current size less than 20% larger than previous size.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   630
  if (UseSuperWord && cl->node_count_before_unroll() > 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   631
      future_unroll_ct > LoopUnrollMin &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   632
      (future_unroll_ct - 1) * 10.0 > cl->profile_trip_cnt() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   633
      1.2 * cl->node_count_before_unroll() < (double)_body.size()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   634
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   635
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   636
489c9b5090e2 Initial load
duke
parents:
diff changeset
   637
  Node *init_n = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   638
  Node *limit_n = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   639
  // Non-constant bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   640
  // Protect against over-unrolling when init or/and limit are not constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   641
  // (so that trip_count's init value is maxint) but iv range is known.
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   642
  if (init_n   == NULL || !init_n->is_Con()  ||
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   643
      limit_n  == NULL || !limit_n->is_Con()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   644
    Node* phi = cl->phi();
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   645
    if (phi != NULL) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   646
      assert(phi->is_Phi() && phi->in(0) == _head, "Counted loop should have iv phi.");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   647
      const TypeInt* iv_type = phase->_igvn.type(phi)->is_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   648
      int next_stride = cl->stride_con() * 2; // stride after this unroll
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   649
      if (next_stride > 0) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   650
        if (iv_type->_lo + next_stride <= iv_type->_lo || // overflow
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   651
            iv_type->_lo + next_stride >  iv_type->_hi) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   652
          return false;  // over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   653
        }
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   654
      } else if (next_stride < 0) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   655
        if (iv_type->_hi + next_stride >= iv_type->_hi || // overflow
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   656
            iv_type->_hi + next_stride <  iv_type->_lo) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   657
          return false;  // over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   658
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   659
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   660
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   661
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   662
489c9b5090e2 Initial load
duke
parents:
diff changeset
   663
  // Adjust body_size to determine if we unroll or not
489c9b5090e2 Initial load
duke
parents:
diff changeset
   664
  uint body_size = _body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   665
  // Key test to unroll CaffeineMark's Logic test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   666
  int xors_in_loop = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   667
  // Also count ModL, DivL and MulL which expand mightly
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   668
  for (uint k = 0; k < _body.size(); k++) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   669
    Node* n = _body.at(k);
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   670
    switch (n->Opcode()) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   671
      case Op_XorI: xors_in_loop++; break; // CaffeineMark's Logic test
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   672
      case Op_ModL: body_size += 30; break;
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   673
      case Op_DivL: body_size += 30; break;
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   674
      case Op_MulL: body_size += 10; break;
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   675
      case Op_StrComp:
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   676
      case Op_StrEquals:
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   677
      case Op_StrIndexOf:
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   678
      case Op_AryEq: {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   679
        // Do not unroll a loop with String intrinsics code.
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   680
        // String intrinsics are large and have loops.
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   681
        return false;
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   682
      }
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   683
    } // switch
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   684
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   685
489c9b5090e2 Initial load
duke
parents:
diff changeset
   686
  // Check for being too big
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   687
  if (body_size > (uint)LoopUnrollLimit) {
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   688
    if (xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   689
    // Normal case: loop too big
489c9b5090e2 Initial load
duke
parents:
diff changeset
   690
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   691
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   692
489c9b5090e2 Initial load
duke
parents:
diff changeset
   693
  // Check for stride being a small enough constant
8879
e6ae9a7aa72a 7029152: Ideal nodes for String intrinsics miss memory edge optimization
kvn
parents: 8870
diff changeset
   694
  if (abs(cl->stride_con()) > (1<<3)) return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   695
489c9b5090e2 Initial load
duke
parents:
diff changeset
   696
  // Unroll once!  (Each trip will soon do double iterations)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   697
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   698
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   699
489c9b5090e2 Initial load
duke
parents:
diff changeset
   700
//------------------------------policy_align-----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   701
// Return TRUE or FALSE if the loop should be cache-line aligned.  Gather the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   702
// expression that does the alignment.  Note that only one array base can be
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1500
diff changeset
   703
// aligned in a loop (unless the VM guarantees mutual alignment).  Note that
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   704
// if we vectorize short memory ops into longer memory ops, we may want to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   705
// increase alignment.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   706
bool IdealLoopTree::policy_align( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   707
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   708
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   709
489c9b5090e2 Initial load
duke
parents:
diff changeset
   710
//------------------------------policy_range_check-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   711
// Return TRUE or FALSE if the loop should be range-check-eliminated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   712
// Actually we do iteration-splitting, a more powerful form of RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   713
bool IdealLoopTree::policy_range_check( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   714
  if( !RangeCheckElimination ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   715
489c9b5090e2 Initial load
duke
parents:
diff changeset
   716
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   717
  // If we unrolled with no intention of doing RCE and we later
489c9b5090e2 Initial load
duke
parents:
diff changeset
   718
  // changed our minds, we got no pre-loop.  Either we need to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   719
  // make a new pre-loop, or we gotta disallow RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   720
  if( cl->is_main_no_pre_loop() ) return false; // Disallowed for now.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   721
  Node *trip_counter = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   722
489c9b5090e2 Initial load
duke
parents:
diff changeset
   723
  // Check loop body for tests of trip-counter plus loop-invariant vs
489c9b5090e2 Initial load
duke
parents:
diff changeset
   724
  // loop-invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   725
  for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   726
    Node *iff = _body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   727
    if( iff->Opcode() == Op_If ) { // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   728
489c9b5090e2 Initial load
duke
parents:
diff changeset
   729
      // Comparing trip+off vs limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
   730
      Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   731
      if( bol->req() != 2 ) continue; // dead constant test
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   732
      if (!bol->is_Bool()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   733
        assert(UseLoopPredicate && bol->Opcode() == Op_Conv2B, "predicate check only");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   734
        continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   735
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   736
      Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   737
489c9b5090e2 Initial load
duke
parents:
diff changeset
   738
      Node *rc_exp = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   739
      Node *limit = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   740
489c9b5090e2 Initial load
duke
parents:
diff changeset
   741
      Node *limit_c = phase->get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   742
      if( limit_c == phase->C->top() )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   743
        return false;           // Found dead test on live IF?  No RCE!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   744
      if( is_member(phase->get_loop(limit_c) ) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   745
        // Compare might have operands swapped; commute them
489c9b5090e2 Initial load
duke
parents:
diff changeset
   746
        rc_exp = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   747
        limit  = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   748
        limit_c = phase->get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   749
        if( is_member(phase->get_loop(limit_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   750
          continue;             // Both inputs are loop varying; cannot RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
   751
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   752
489c9b5090e2 Initial load
duke
parents:
diff changeset
   753
      if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, NULL, NULL)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   754
        continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   755
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   756
      // Yeah!  Found a test like 'trip+off vs limit'
489c9b5090e2 Initial load
duke
parents:
diff changeset
   757
      // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   758
      // we need loop unswitching instead of iteration splitting.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   759
      if( is_loop_exit(iff) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   760
        return true;            // Found reason to split iterations
489c9b5090e2 Initial load
duke
parents:
diff changeset
   761
    } // End of is IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
   762
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   763
489c9b5090e2 Initial load
duke
parents:
diff changeset
   764
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   765
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   766
489c9b5090e2 Initial load
duke
parents:
diff changeset
   767
//------------------------------policy_peel_only-------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   768
// Return TRUE or FALSE if the loop should NEVER be RCE'd or aligned.  Useful
489c9b5090e2 Initial load
duke
parents:
diff changeset
   769
// for unrolling loops with NO array accesses.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   770
bool IdealLoopTree::policy_peel_only( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   771
489c9b5090e2 Initial load
duke
parents:
diff changeset
   772
  for( uint i = 0; i < _body.size(); i++ )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   773
    if( _body[i]->is_Mem() )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   774
      return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   775
489c9b5090e2 Initial load
duke
parents:
diff changeset
   776
  // No memory accesses at all!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   777
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   778
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   779
489c9b5090e2 Initial load
duke
parents:
diff changeset
   780
//------------------------------clone_up_backedge_goo--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   781
// If Node n lives in the back_ctrl block and cannot float, we clone a private
489c9b5090e2 Initial load
duke
parents:
diff changeset
   782
// version of n in preheader_ctrl block and return that, otherwise return n.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   783
Node *PhaseIdealLoop::clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   784
  if( get_ctrl(n) != back_ctrl ) return n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   785
489c9b5090e2 Initial load
duke
parents:
diff changeset
   786
  Node *x = NULL;               // If required, a clone of 'n'
489c9b5090e2 Initial load
duke
parents:
diff changeset
   787
  // Check for 'n' being pinned in the backedge.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   788
  if( n->in(0) && n->in(0) == back_ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   789
    x = n->clone();             // Clone a copy of 'n' to preheader
489c9b5090e2 Initial load
duke
parents:
diff changeset
   790
    x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
489c9b5090e2 Initial load
duke
parents:
diff changeset
   791
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   792
489c9b5090e2 Initial load
duke
parents:
diff changeset
   793
  // Recursive fixup any other input edges into x.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   794
  // If there are no changes we can just return 'n', otherwise
489c9b5090e2 Initial load
duke
parents:
diff changeset
   795
  // we need to clone a private copy and change it.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   796
  for( uint i = 1; i < n->req(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   797
    Node *g = clone_up_backedge_goo( back_ctrl, preheader_ctrl, n->in(i) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   798
    if( g != n->in(i) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   799
      if( !x )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   800
        x = n->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   801
      x->set_req(i, g);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   802
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   803
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   804
  if( x ) {                     // x can legally float to pre-header location
489c9b5090e2 Initial load
duke
parents:
diff changeset
   805
    register_new_node( x, preheader_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   806
    return x;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   807
  } else {                      // raise n to cover LCA of uses
489c9b5090e2 Initial load
duke
parents:
diff changeset
   808
    set_ctrl( n, find_non_split_ctrl(back_ctrl->in(0)) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   809
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   810
  return n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   811
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   812
489c9b5090e2 Initial load
duke
parents:
diff changeset
   813
//------------------------------insert_pre_post_loops--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   814
// Insert pre and post loops.  If peel_only is set, the pre-loop can not have
489c9b5090e2 Initial load
duke
parents:
diff changeset
   815
// more iterations added.  It acts as a 'peel' only, no lower-bound RCE, no
489c9b5090e2 Initial load
duke
parents:
diff changeset
   816
// alignment.  Useful to unroll loops that do no array accesses.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   817
void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   818
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   819
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   820
  if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   821
    if (peel_only)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   822
      tty->print("PeelMainPost ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   823
    else
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   824
      tty->print("PreMainPost  ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   825
    loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   826
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
   827
#endif
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   828
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   829
489c9b5090e2 Initial load
duke
parents:
diff changeset
   830
  // Find common pieces of the loop being guarded with pre & post loops
489c9b5090e2 Initial load
duke
parents:
diff changeset
   831
  CountedLoopNode *main_head = loop->_head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   832
  assert( main_head->is_normal_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   833
  CountedLoopEndNode *main_end = main_head->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   834
  assert( main_end->outcnt() == 2, "1 true, 1 false path only" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   835
  uint dd_main_head = dom_depth(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   836
  uint max = main_head->outcnt();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   837
489c9b5090e2 Initial load
duke
parents:
diff changeset
   838
  Node *pre_header= main_head->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   839
  Node *init      = main_head->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   840
  Node *incr      = main_end ->incr();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   841
  Node *limit     = main_end ->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   842
  Node *stride    = main_end ->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   843
  Node *cmp       = main_end ->cmp_node();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   844
  BoolTest::mask b_test = main_end->test_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   845
489c9b5090e2 Initial load
duke
parents:
diff changeset
   846
  // Need only 1 user of 'bol' because I will be hacking the loop bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   847
  Node *bol = main_end->in(CountedLoopEndNode::TestValue);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   848
  if( bol->outcnt() != 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   849
    bol = bol->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   850
    register_new_node(bol,main_end->in(CountedLoopEndNode::TestControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   851
    _igvn.hash_delete(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   852
    main_end->set_req(CountedLoopEndNode::TestValue, bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   853
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   854
  // Need only 1 user of 'cmp' because I will be hacking the loop bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   855
  if( cmp->outcnt() != 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   856
    cmp = cmp->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   857
    register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   858
    _igvn.hash_delete(bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   859
    bol->set_req(1, cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   860
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   861
489c9b5090e2 Initial load
duke
parents:
diff changeset
   862
  //------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   863
  // Step A: Create Post-Loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   864
  Node* main_exit = main_end->proj_out(false);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   865
  assert( main_exit->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   866
  int dd_main_exit = dom_depth(main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   867
489c9b5090e2 Initial load
duke
parents:
diff changeset
   868
  // Step A1: Clone the loop body.  The clone becomes the post-loop.  The main
489c9b5090e2 Initial load
duke
parents:
diff changeset
   869
  // loop pre-header illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   870
  clone_loop( loop, old_new, dd_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   871
  assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   872
  CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   873
  post_head->set_post_loop(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   874
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   875
  // Reduce the post-loop trip count.
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   876
  CountedLoopEndNode* post_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   877
  post_end->_prob = PROB_FAIR;
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   878
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   879
  // Build the main-loop normal exit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   880
  IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   881
  _igvn.register_new_node_with_optimizer( new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   882
  set_idom(new_main_exit, main_end, dd_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   883
  set_loop(new_main_exit, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   884
489c9b5090e2 Initial load
duke
parents:
diff changeset
   885
  // Step A2: Build a zero-trip guard for the post-loop.  After leaving the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   886
  // main-loop, the post-loop may not execute at all.  We 'opaque' the incr
489c9b5090e2 Initial load
duke
parents:
diff changeset
   887
  // (the main-loop trip-counter exit value) because we will be changing
489c9b5090e2 Initial load
duke
parents:
diff changeset
   888
  // the exit value (via unrolling) so we cannot constant-fold away the zero
489c9b5090e2 Initial load
duke
parents:
diff changeset
   889
  // trip guard until all unrolling is done.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   890
  Node *zer_opaq = new (C, 2) Opaque1Node(C, incr);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   891
  Node *zer_cmp  = new (C, 3) CmpINode( zer_opaq, limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   892
  Node *zer_bol  = new (C, 2) BoolNode( zer_cmp, b_test );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   893
  register_new_node( zer_opaq, new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   894
  register_new_node( zer_cmp , new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   895
  register_new_node( zer_bol , new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   896
489c9b5090e2 Initial load
duke
parents:
diff changeset
   897
  // Build the IfNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
   898
  IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   899
  _igvn.register_new_node_with_optimizer( zer_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   900
  set_idom(zer_iff, new_main_exit, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   901
  set_loop(zer_iff, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   902
489c9b5090e2 Initial load
duke
parents:
diff changeset
   903
  // Plug in the false-path, taken if we need to skip post-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   904
  _igvn.hash_delete( main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   905
  main_exit->set_req(0, zer_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   906
  _igvn._worklist.push(main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   907
  set_idom(main_exit, zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   908
  set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   909
  // Make the true-path, must enter the post loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   910
  Node *zer_taken = new (C, 1) IfTrueNode( zer_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   911
  _igvn.register_new_node_with_optimizer( zer_taken );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   912
  set_idom(zer_taken, zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   913
  set_loop(zer_taken, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   914
  // Plug in the true path
489c9b5090e2 Initial load
duke
parents:
diff changeset
   915
  _igvn.hash_delete( post_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   916
  post_head->set_req(LoopNode::EntryControl, zer_taken);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   917
  set_idom(post_head, zer_taken, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   918
489c9b5090e2 Initial load
duke
parents:
diff changeset
   919
  // Step A3: Make the fall-in values to the post-loop come from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   920
  // fall-out values of the main-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   921
  for (DUIterator_Fast imax, i = main_head->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   922
    Node* main_phi = main_head->fast_out(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   923
    if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() >0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   924
      Node *post_phi = old_new[main_phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   925
      Node *fallmain  = clone_up_backedge_goo(main_head->back_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   926
                                              post_head->init_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   927
                                              main_phi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   928
      _igvn.hash_delete(post_phi);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   929
      post_phi->set_req( LoopNode::EntryControl, fallmain );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   930
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   931
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   932
489c9b5090e2 Initial load
duke
parents:
diff changeset
   933
  // Update local caches for next stanza
489c9b5090e2 Initial load
duke
parents:
diff changeset
   934
  main_exit = new_main_exit;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   935
489c9b5090e2 Initial load
duke
parents:
diff changeset
   936
489c9b5090e2 Initial load
duke
parents:
diff changeset
   937
  //------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   938
  // Step B: Create Pre-Loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   939
489c9b5090e2 Initial load
duke
parents:
diff changeset
   940
  // Step B1: Clone the loop body.  The clone becomes the pre-loop.  The main
489c9b5090e2 Initial load
duke
parents:
diff changeset
   941
  // loop pre-header illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   942
  clone_loop( loop, old_new, dd_main_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   943
  CountedLoopNode*    pre_head = old_new[main_head->_idx]->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   944
  CountedLoopEndNode* pre_end  = old_new[main_end ->_idx]->as_CountedLoopEnd();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   945
  pre_head->set_pre_loop(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   946
  Node *pre_incr = old_new[incr->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   947
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   948
  // Reduce the pre-loop trip count.
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   949
  pre_end->_prob = PROB_FAIR;
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   950
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   951
  // Find the pre-loop normal exit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   952
  Node* pre_exit = pre_end->proj_out(false);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   953
  assert( pre_exit->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   954
  IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   955
  _igvn.register_new_node_with_optimizer( new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   956
  set_idom(new_pre_exit, pre_end, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   957
  set_loop(new_pre_exit, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   958
489c9b5090e2 Initial load
duke
parents:
diff changeset
   959
  // Step B2: Build a zero-trip guard for the main-loop.  After leaving the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   960
  // pre-loop, the main-loop may not execute at all.  Later in life this
489c9b5090e2 Initial load
duke
parents:
diff changeset
   961
  // zero-trip guard will become the minimum-trip guard when we unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   962
  // the main-loop.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   963
  Node *min_opaq = new (C, 2) Opaque1Node(C, limit);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   964
  Node *min_cmp  = new (C, 3) CmpINode( pre_incr, min_opaq );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   965
  Node *min_bol  = new (C, 2) BoolNode( min_cmp, b_test );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   966
  register_new_node( min_opaq, new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   967
  register_new_node( min_cmp , new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   968
  register_new_node( min_bol , new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   969
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   970
  // Build the IfNode (assume the main-loop is executed always).
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   971
  IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_ALWAYS, COUNT_UNKNOWN );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   972
  _igvn.register_new_node_with_optimizer( min_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   973
  set_idom(min_iff, new_pre_exit, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   974
  set_loop(min_iff, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   975
489c9b5090e2 Initial load
duke
parents:
diff changeset
   976
  // Plug in the false-path, taken if we need to skip main-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   977
  _igvn.hash_delete( pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   978
  pre_exit->set_req(0, min_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   979
  set_idom(pre_exit, min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   980
  set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   981
  // Make the true-path, must enter the main loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   982
  Node *min_taken = new (C, 1) IfTrueNode( min_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   983
  _igvn.register_new_node_with_optimizer( min_taken );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   984
  set_idom(min_taken, min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   985
  set_loop(min_taken, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   986
  // Plug in the true path
489c9b5090e2 Initial load
duke
parents:
diff changeset
   987
  _igvn.hash_delete( main_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   988
  main_head->set_req(LoopNode::EntryControl, min_taken);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   989
  set_idom(main_head, min_taken, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   990
489c9b5090e2 Initial load
duke
parents:
diff changeset
   991
  // Step B3: Make the fall-in values to the main-loop come from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   992
  // fall-out values of the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   993
  for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   994
    Node* main_phi = main_head->fast_out(i2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   995
    if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   996
      Node *pre_phi = old_new[main_phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   997
      Node *fallpre  = clone_up_backedge_goo(pre_head->back_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   998
                                             main_head->init_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   999
                                             pre_phi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1000
      _igvn.hash_delete(main_phi);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1001
      main_phi->set_req( LoopNode::EntryControl, fallpre );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1002
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1003
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1004
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1005
  // Step B4: Shorten the pre-loop to run only 1 iteration (for now).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1006
  // RCE and alignment may change this later.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1007
  Node *cmp_end = pre_end->cmp_node();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1008
  assert( cmp_end->in(2) == limit, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1009
  Node *pre_limit = new (C, 3) AddINode( init, stride );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1010
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1011
  // Save the original loop limit in this Opaque1 node for
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1012
  // use by range check elimination.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
  1013
  Node *pre_opaq  = new (C, 3) Opaque1Node(C, pre_limit, limit);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1014
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1015
  register_new_node( pre_limit, pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1016
  register_new_node( pre_opaq , pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1017
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1018
  // Since no other users of pre-loop compare, I can hack limit directly
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1019
  assert( cmp_end->outcnt() == 1, "no other users" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1020
  _igvn.hash_delete(cmp_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1021
  cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1022
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1023
  // Special case for not-equal loop bounds:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1024
  // Change pre loop test, main loop test, and the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1025
  // main loop guard test to use lt or gt depending on stride
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1026
  // direction:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1027
  // positive stride use <
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1028
  // negative stride use >
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1029
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1030
  if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1031
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1032
    BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1033
    // Modify pre loop end condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1034
    Node* pre_bol = pre_end->in(CountedLoopEndNode::TestValue)->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1035
    BoolNode* new_bol0 = new (C, 2) BoolNode(pre_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1036
    register_new_node( new_bol0, pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1037
    _igvn.hash_delete(pre_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1038
    pre_end->set_req(CountedLoopEndNode::TestValue, new_bol0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1039
    // Modify main loop guard condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1040
    assert(min_iff->in(CountedLoopEndNode::TestValue) == min_bol, "guard okay");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1041
    BoolNode* new_bol1 = new (C, 2) BoolNode(min_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1042
    register_new_node( new_bol1, new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1043
    _igvn.hash_delete(min_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1044
    min_iff->set_req(CountedLoopEndNode::TestValue, new_bol1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1045
    // Modify main loop end condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1046
    BoolNode* main_bol = main_end->in(CountedLoopEndNode::TestValue)->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1047
    BoolNode* new_bol2 = new (C, 2) BoolNode(main_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1048
    register_new_node( new_bol2, main_end->in(CountedLoopEndNode::TestControl) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1049
    _igvn.hash_delete(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1050
    main_end->set_req(CountedLoopEndNode::TestValue, new_bol2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1051
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1052
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1053
  // Flag main loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1054
  main_head->set_main_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1055
  if( peel_only ) main_head->set_main_no_pre_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1056
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1057
  // It's difficult to be precise about the trip-counts
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1058
  // for the pre/post loops.  They are usually very short,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1059
  // so guess that 4 trips is a reasonable value.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1060
  post_head->set_profile_trip_cnt(4.0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1061
  pre_head->set_profile_trip_cnt(4.0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1062
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1063
  // Now force out all loop-invariant dominating tests.  The optimizer
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1064
  // finds some, but we _know_ they are all useless.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1065
  peeled_dom_test_elim(loop,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1066
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1067
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1068
//------------------------------is_invariant-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1069
// Return true if n is invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1070
bool IdealLoopTree::is_invariant(Node* n) const {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1071
  Node *n_c = _phase->has_ctrl(n) ? _phase->get_ctrl(n) : n;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1072
  if (n_c->is_top()) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1073
  return !is_member(_phase->get_loop(n_c));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1074
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1075
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1076
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1077
//------------------------------do_unroll--------------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1078
// Unroll the loop body one step - make each trip do 2 iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1079
void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ) {
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1080
  assert(LoopUnrollLimit, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1081
  CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1082
  CountedLoopEndNode *loop_end = loop_head->loopexit();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1083
  assert(loop_end, "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1084
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1085
  if (PrintOpto && VerifyLoopOptimizations) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1086
    tty->print("Unrolling ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1087
    loop->dump_head();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1088
  } else if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1089
    tty->print("Unroll     %d ", loop_head->unrolled_count()*2);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1090
    loop->dump_head();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1091
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1092
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1093
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1094
  // Remember loop node count before unrolling to detect
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1095
  // if rounds of unroll,optimize are making progress
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1096
  loop_head->set_node_count_before_unroll(loop->_body.size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1097
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1098
  Node *ctrl  = loop_head->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1099
  Node *limit = loop_head->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1100
  Node *init  = loop_head->init_trip();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1101
  Node *stride = loop_head->stride();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1102
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1103
  Node *opaq = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1104
  if( adjust_min_trip ) {       // If not maximally unrolling, need adjustment
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1105
    assert( loop_head->is_main_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1106
    assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1107
    Node *iff = ctrl->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1108
    assert( iff->Opcode() == Op_If, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1109
    Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1110
    assert( bol->Opcode() == Op_Bool, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1111
    Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1112
    assert( cmp->Opcode() == Op_CmpI, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1113
    opaq = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1114
    // Occasionally it's possible for a pre-loop Opaque1 node to be
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1115
    // optimized away and then another round of loop opts attempted.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1116
    // We can not optimize this particular loop in that case.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1117
    if( opaq->Opcode() != Op_Opaque1 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1118
      return;                   // Cannot find pre-loop!  Bail out!
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1119
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1120
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1121
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1122
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1123
  // Adjust max trip count. The trip count is intentionally rounded
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1124
  // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1125
  // the main, unrolled, part of the loop will never execute as it is protected
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1126
  // by the min-trip test.  See bug 4834191 for a case where we over-unrolled
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1127
  // and later determined that part of the unrolled loop was dead.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1128
  loop_head->set_trip_count(loop_head->trip_count() / 2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1129
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1130
  // Double the count of original iterations in the unrolled loop body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1131
  loop_head->double_unrolled_count();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1132
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1133
  // -----------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1134
  // Step 2: Cut back the trip counter for an unroll amount of 2.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1135
  // Loop will normally trip (limit - init)/stride_con.  Since it's a
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1136
  // CountedLoop this is exact (stride divides limit-init exactly).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1137
  // We are going to double the loop body, so we want to knock off any
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1138
  // odd iteration: (trip_cnt & ~1).  Then back compute a new limit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1139
  Node *span = new (C, 3) SubINode( limit, init );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1140
  register_new_node( span, ctrl );
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1141
  Node *trip = new (C, 3) DivINode( 0, span, stride );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1142
  register_new_node( trip, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1143
  Node *mtwo = _igvn.intcon(-2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1144
  set_ctrl(mtwo, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1145
  Node *rond = new (C, 3) AndINode( trip, mtwo );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1146
  register_new_node( rond, ctrl );
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1147
  Node *spn2 = new (C, 3) MulINode( rond, stride );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1148
  register_new_node( spn2, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1149
  Node *lim2 = new (C, 3) AddINode( spn2, init );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1150
  register_new_node( lim2, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1151
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1152
  // Hammer in the new limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1153
  Node *ctrl2 = loop_end->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1154
  Node *cmp2 = new (C, 3) CmpINode( loop_head->incr(), lim2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1155
  register_new_node( cmp2, ctrl2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1156
  Node *bol2 = new (C, 2) BoolNode( cmp2, loop_end->test_trip() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1157
  register_new_node( bol2, ctrl2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1158
  _igvn.hash_delete(loop_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1159
  loop_end->set_req(CountedLoopEndNode::TestValue, bol2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1160
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1161
  // Step 3: Find the min-trip test guaranteed before a 'main' loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1162
  // Make it a 1-trip test (means at least 2 trips).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1163
  if( adjust_min_trip ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1164
    // Guard test uses an 'opaque' node which is not shared.  Hence I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1165
    // can edit it's inputs directly.  Hammer in the new limit for the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1166
    // minimum-trip guard.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1167
    assert( opaq->outcnt() == 1, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1168
    _igvn.hash_delete(opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1169
    opaq->set_req(1, lim2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1170
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1171
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1172
  // ---------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1173
  // Step 4: Clone the loop body.  Move it inside the loop.  This loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1174
  // represents the odd iterations; since the loop trips an even number of
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1175
  // times its backedge is never taken.  Kill the backedge.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1176
  uint dd = dom_depth(loop_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1177
  clone_loop( loop, old_new, dd );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1178
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1179
  // Make backedges of the clone equal to backedges of the original.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1180
  // Make the fall-in from the original come from the fall-out of the clone.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1181
  for (DUIterator_Fast jmax, j = loop_head->fast_outs(jmax); j < jmax; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1182
    Node* phi = loop_head->fast_out(j);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1183
    if( phi->is_Phi() && phi->in(0) == loop_head && phi->outcnt() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1184
      Node *newphi = old_new[phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1185
      _igvn.hash_delete( phi );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1186
      _igvn.hash_delete( newphi );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1187
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1188
      phi   ->set_req(LoopNode::   EntryControl, newphi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1189
      newphi->set_req(LoopNode::LoopBackControl, phi   ->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1190
      phi   ->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1191
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1192
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1193
  Node *clone_head = old_new[loop_head->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1194
  _igvn.hash_delete( clone_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1195
  loop_head ->set_req(LoopNode::   EntryControl, clone_head->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1196
  clone_head->set_req(LoopNode::LoopBackControl, loop_head ->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1197
  loop_head ->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1198
  loop->_head = clone_head;     // New loop header
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1199
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1200
  set_idom(loop_head,  loop_head ->in(LoopNode::EntryControl), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1201
  set_idom(clone_head, clone_head->in(LoopNode::EntryControl), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1202
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1203
  // Kill the clone's backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1204
  Node *newcle = old_new[loop_end->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1205
  _igvn.hash_delete( newcle );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1206
  Node *one = _igvn.intcon(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1207
  set_ctrl(one, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1208
  newcle->set_req(1, one);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1209
  // Force clone into same loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1210
  uint max = loop->_body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1211
  for( uint k = 0; k < max; k++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1212
    Node *old = loop->_body.at(k);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1213
    Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1214
    loop->_body.push(nnn);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1215
    if (!has_ctrl(old))
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1216
      set_loop(nnn, loop);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1217
  }
1399
9648dfd4ce09 6384206: Phis which are later unneeded are impairing our ability to inline based on static types
never
parents: 781
diff changeset
  1218
9648dfd4ce09 6384206: Phis which are later unneeded are impairing our ability to inline based on static types
never
parents: 781
diff changeset
  1219
  loop->record_for_igvn();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1220
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1221
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1222
//------------------------------do_maximally_unroll----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1223
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1224
void PhaseIdealLoop::do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1225
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1226
  assert(cl->trip_count() > 0, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1227
#ifndef PRODUCT
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1228
  if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1229
    tty->print("MaxUnroll  %d ", cl->trip_count());
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1230
    loop->dump_head();
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1231
  }
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1232
#endif
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1233
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1234
  // If loop is tripping an odd number of times, peel odd iteration
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1235
  if ((cl->trip_count() & 1) == 1) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1236
    do_peeling(loop, old_new);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1237
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1238
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1239
  // Now its tripping an even number of times remaining.  Double loop body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1240
  // Do not adjust pre-guards; they are not needed and do not exist.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1241
  if (cl->trip_count() > 0) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1242
    do_unroll(loop, old_new, false);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1243
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1244
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1245
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1246
//------------------------------dominates_backedge---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1247
// Returns true if ctrl is executed on every complete iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1248
bool IdealLoopTree::dominates_backedge(Node* ctrl) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1249
  assert(ctrl->is_CFG(), "must be control");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1250
  Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1251
  return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1252
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1253
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1254
//------------------------------add_constraint---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1255
// Constrain the main loop iterations so the condition:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1256
//    scale_con * I + offset  <  limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1257
// always holds true.  That is, either increase the number of iterations in
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1258
// the pre-loop or the post-loop until the condition holds true in the main
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1259
// loop.  Stride, scale, offset and limit are all loop invariant.  Further,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1260
// stride and scale are constants (offset and limit often are).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1261
void PhaseIdealLoop::add_constraint( int stride_con, int scale_con, Node *offset, Node *limit, Node *pre_ctrl, Node **pre_limit, Node **main_limit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1262
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1263
  // Compute "I :: (limit-offset)/scale_con"
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1264
  Node *con = new (C, 3) SubINode( limit, offset );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1265
  register_new_node( con, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1266
  Node *scale = _igvn.intcon(scale_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1267
  set_ctrl(scale, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1268
  Node *X = new (C, 3) DivINode( 0, con, scale );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1269
  register_new_node( X, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1270
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1271
  // For positive stride, the pre-loop limit always uses a MAX function
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1272
  // and the main loop a MIN function.  For negative stride these are
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1273
  // reversed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1274
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1275
  // Also for positive stride*scale the affine function is increasing, so the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1276
  // pre-loop must check for underflow and the post-loop for overflow.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1277
  // Negative stride*scale reverses this; pre-loop checks for overflow and
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1278
  // post-loop for underflow.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1279
  if( stride_con*scale_con > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1280
    // Compute I < (limit-offset)/scale_con
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1281
    // Adjust main-loop last iteration to be MIN/MAX(main_loop,X)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1282
    *main_limit = (stride_con > 0)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1283
      ? (Node*)(new (C, 3) MinINode( *main_limit, X ))
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1284
      : (Node*)(new (C, 3) MaxINode( *main_limit, X ));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1285
    register_new_node( *main_limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1286
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1287
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1288
    // Compute (limit-offset)/scale_con + SGN(-scale_con) <= I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1289
    // Add the negation of the main-loop constraint to the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1290
    // See footnote [++] below for a derivation of the limit expression.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1291
    Node *incr = _igvn.intcon(scale_con > 0 ? -1 : 1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1292
    set_ctrl(incr, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1293
    Node *adj = new (C, 3) AddINode( X, incr );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1294
    register_new_node( adj, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1295
    *pre_limit = (scale_con > 0)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1296
      ? (Node*)new (C, 3) MinINode( *pre_limit, adj )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1297
      : (Node*)new (C, 3) MaxINode( *pre_limit, adj );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1298
    register_new_node( *pre_limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1299
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1300
//   [++] Here's the algebra that justifies the pre-loop limit expression:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1301
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1302
//   NOT( scale_con * I + offset  <  limit )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1303
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1304
//   scale_con * I + offset  >=  limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1305
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1306
//   SGN(scale_con) * I  >=  (limit-offset)/|scale_con|
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1307
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1308
//   (limit-offset)/|scale_con|   <=  I * SGN(scale_con)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1309
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1310
//   (limit-offset)/|scale_con|-1  <  I * SGN(scale_con)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1311
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1312
//   ( if (scale_con > 0) /*common case*/
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1313
//       (limit-offset)/scale_con - 1  <  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1314
//     else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1315
//       (limit-offset)/scale_con + 1  >  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1316
//    )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1317
//   ( if (scale_con > 0) /*common case*/
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1318
//       (limit-offset)/scale_con + SGN(-scale_con)  <  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1319
//     else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1320
//       (limit-offset)/scale_con + SGN(-scale_con)  >  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1321
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1322
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1323
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1324
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1325
//------------------------------is_scaled_iv---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1326
// Return true if exp is a constant times an induction var
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1327
bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1328
  if (exp == iv) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1329
    if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1330
      *p_scale = 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1331
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1332
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1333
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1334
  int opc = exp->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1335
  if (opc == Op_MulI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1336
    if (exp->in(1) == iv && exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1337
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1338
        *p_scale = exp->in(2)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1339
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1340
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1341
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1342
    if (exp->in(2) == iv && exp->in(1)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1343
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1344
        *p_scale = exp->in(1)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1345
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1346
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1347
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1348
  } else if (opc == Op_LShiftI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1349
    if (exp->in(1) == iv && exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1350
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1351
        *p_scale = 1 << exp->in(2)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1352
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1353
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1354
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1355
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1356
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1357
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1358
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1359
//-----------------------------is_scaled_iv_plus_offset------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1360
// Return true if exp is a simple induction variable expression: k1*iv + (invar + k2)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1361
bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1362
  if (is_scaled_iv(exp, iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1363
    if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1364
      Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1365
      set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1366
      *p_offset = zero;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1367
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1368
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1369
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1370
  int opc = exp->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1371
  if (opc == Op_AddI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1372
    if (is_scaled_iv(exp->in(1), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1373
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1374
        *p_offset = exp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1375
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1376
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1377
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1378
    if (exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1379
      Node* offset2 = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1380
      if (depth < 2 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1381
          is_scaled_iv_plus_offset(exp->in(1), iv, p_scale,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1382
                                   p_offset != NULL ? &offset2 : NULL, depth+1)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1383
        if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1384
          Node *ctrl_off2 = get_ctrl(offset2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1385
          Node* offset = new (C, 3) AddINode(offset2, exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1386
          register_new_node(offset, ctrl_off2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1387
          *p_offset = offset;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1388
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1389
        return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1390
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1391
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1392
  } else if (opc == Op_SubI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1393
    if (is_scaled_iv(exp->in(1), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1394
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1395
        Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1396
        set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1397
        Node *ctrl_off = get_ctrl(exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1398
        Node* offset = new (C, 3) SubINode(zero, exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1399
        register_new_node(offset, ctrl_off);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1400
        *p_offset = offset;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1401
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1402
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1403
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1404
    if (is_scaled_iv(exp->in(2), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1405
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1406
        *p_scale *= -1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1407
        *p_offset = exp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1408
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1409
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1410
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1411
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1412
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1413
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1414
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1415
//------------------------------do_range_check---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1416
// Eliminate range-checks and other trip-counter vs loop-invariant tests.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1417
void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1418
#ifndef PRODUCT
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1419
  if (PrintOpto && VerifyLoopOptimizations) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1420
    tty->print("Range Check Elimination ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1421
    loop->dump_head();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1422
  } else if (TraceLoopOpts) {
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1423
    tty->print("RangeCheck   ");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1424
    loop->dump_head();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1425
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1426
#endif
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1427
  assert(RangeCheckElimination, "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1428
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1429
  assert(cl->is_main_loop(), "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1430
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1431
  // protect against stride not being a constant
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1432
  if (!cl->stride_is_con())
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1433
    return;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1434
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1435
  // Find the trip counter; we are iteration splitting based on it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1436
  Node *trip_counter = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1437
  // Find the main loop limit; we will trim it's iterations
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1438
  // to not ever trip end tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1439
  Node *main_limit = cl->limit();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1440
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1441
  // Need to find the main-loop zero-trip guard
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1442
  Node *ctrl  = cl->in(LoopNode::EntryControl);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1443
  assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1444
  Node *iffm = ctrl->in(0);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1445
  assert(iffm->Opcode() == Op_If, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1446
  Node *bolzm = iffm->in(1);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1447
  assert(bolzm->Opcode() == Op_Bool, "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1448
  Node *cmpzm = bolzm->in(1);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1449
  assert(cmpzm->is_Cmp(), "");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1450
  Node *opqzm = cmpzm->in(2);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1451
  // Can not optimize a loop if pre-loop Opaque1 node is optimized
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1452
  // away and then another round of loop opts attempted.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1453
  if (opqzm->Opcode() != Op_Opaque1)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1454
    return;
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1455
  assert(opqzm->in(1) == main_limit, "do not understand situation");
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1456
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1457
  // Find the pre-loop limit; we will expand it's iterations to
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1458
  // not ever trip low tests.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1459
  Node *p_f = iffm->in(0);
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1460
  assert(p_f->Opcode() == Op_IfFalse, "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1461
  CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1462
  assert(pre_end->loopnode()->is_pre_loop(), "");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1463
  Node *pre_opaq1 = pre_end->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1464
  // Occasionally it's possible for a pre-loop Opaque1 node to be
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1465
  // optimized away and then another round of loop opts attempted.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1466
  // We can not optimize this particular loop in that case.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1467
  if (pre_opaq1->Opcode() != Op_Opaque1)
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1468
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1469
  Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1470
  Node *pre_limit = pre_opaq->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1471
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1472
  // Where do we put new limit calculations
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1473
  Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1474
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1475
  // Ensure the original loop limit is available from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1476
  // pre-loop Opaque1 node.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1477
  Node *orig_limit = pre_opaq->original_loop_limit();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1478
  if (orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP)
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1479
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1480
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1481
  // Must know if its a count-up or count-down loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1482
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1483
  int stride_con = cl->stride_con();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1484
  Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1485
  Node *one  = _igvn.intcon(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1486
  set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1487
  set_ctrl(one,  C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1488
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1489
  // Range checks that do not dominate the loop backedge (ie.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1490
  // conditionally executed) can lengthen the pre loop limit beyond
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1491
  // the original loop limit. To prevent this, the pre limit is
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1492
  // (for stride > 0) MINed with the original loop limit (MAXed
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1493
  // stride < 0) when some range_check (rc) is conditionally
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1494
  // executed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1495
  bool conditional_rc = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1496
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1497
  // Check loop body for tests of trip-counter plus loop-invariant vs
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1498
  // loop-invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1499
  for( uint i = 0; i < loop->_body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1500
    Node *iff = loop->_body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1501
    if( iff->Opcode() == Op_If ) { // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1502
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1503
      // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1504
      // we need loop unswitching instead of iteration splitting.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1505
      Node *exit = loop->is_loop_exit(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1506
      if( !exit ) continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1507
      int flip = (exit->Opcode() == Op_IfTrue) ? 1 : 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1508
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1509
      // Get boolean condition to test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1510
      Node *i1 = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1511
      if( !i1->is_Bool() ) continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1512
      BoolNode *bol = i1->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1513
      BoolTest b_test = bol->_test;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1514
      // Flip sense of test if exit condition is flipped
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1515
      if( flip )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1516
        b_test = b_test.negate();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1517
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1518
      // Get compare
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1519
      Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1520
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1521
      // Look for trip_counter + offset vs limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1522
      Node *rc_exp = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1523
      Node *limit  = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1524
      jint scale_con= 1;        // Assume trip counter not scaled
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1525
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1526
      Node *limit_c = get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1527
      if( loop->is_member(get_loop(limit_c) ) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1528
        // Compare might have operands swapped; commute them
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1529
        b_test = b_test.commute();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1530
        rc_exp = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1531
        limit  = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1532
        limit_c = get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1533
        if( loop->is_member(get_loop(limit_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1534
          continue;             // Both inputs are loop varying; cannot RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1535
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1536
      // Here we know 'limit' is loop invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1537
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1538
      // 'limit' maybe pinned below the zero trip test (probably from a
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1539
      // previous round of rce), in which case, it can't be used in the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1540
      // zero trip test expression which must occur before the zero test's if.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1541
      if( limit_c == ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1542
        continue;  // Don't rce this check but continue looking for other candidates.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1543
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1544
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1545
      // Check for scaled induction variable plus an offset
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1546
      Node *offset = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1547
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1548
      if (!is_scaled_iv_plus_offset(rc_exp, trip_counter, &scale_con, &offset)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1549
        continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1550
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1551
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1552
      Node *offset_c = get_ctrl(offset);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1553
      if( loop->is_member( get_loop(offset_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1554
        continue;               // Offset is not really loop invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1555
      // Here we know 'offset' is loop invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1556
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1557
      // As above for the 'limit', the 'offset' maybe pinned below the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1558
      // zero trip test.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1559
      if( offset_c == ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1560
        continue; // Don't rce this check but continue looking for other candidates.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1561
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1562
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1563
      // At this point we have the expression as:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1564
      //   scale_con * trip_counter + offset :: limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1565
      // where scale_con, offset and limit are loop invariant.  Trip_counter
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1566
      // monotonically increases by stride_con, a constant.  Both (or either)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1567
      // stride_con and scale_con can be negative which will flip about the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1568
      // sense of the test.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1569
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1570
      // Adjust pre and main loop limits to guard the correct iteration set
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1571
      if( cmp->Opcode() == Op_CmpU ) {// Unsigned compare is really 2 tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1572
        if( b_test._test == BoolTest::lt ) { // Range checks always use lt
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1573
          // The overflow limit: scale*I+offset < limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1574
          add_constraint( stride_con, scale_con, offset, limit, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1575
          // The underflow limit: 0 <= scale*I+offset.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1576
          // Some math yields: -scale*I-(offset+1) < 0
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1577
          Node *plus_one = new (C, 3) AddINode( offset, one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1578
          register_new_node( plus_one, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1579
          Node *neg_offset = new (C, 3) SubINode( zero, plus_one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1580
          register_new_node( neg_offset, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1581
          add_constraint( stride_con, -scale_con, neg_offset, zero, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1582
          if (!conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1583
            conditional_rc = !loop->dominates_backedge(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1584
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1585
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1586
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1587
          if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1588
            tty->print_cr("missed RCE opportunity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1589
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1590
          continue;             // In release mode, ignore it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1591
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1592
      } else {                  // Otherwise work on normal compares
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1593
        switch( b_test._test ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1594
        case BoolTest::ge:      // Convert X >= Y to -X <= -Y
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1595
          scale_con = -scale_con;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1596
          offset = new (C, 3) SubINode( zero, offset );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1597
          register_new_node( offset, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1598
          limit  = new (C, 3) SubINode( zero, limit  );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1599
          register_new_node( limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1600
          // Fall into LE case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1601
        case BoolTest::le:      // Convert X <= Y to X < Y+1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1602
          limit = new (C, 3) AddINode( limit, one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1603
          register_new_node( limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1604
          // Fall into LT case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1605
        case BoolTest::lt:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1606
          add_constraint( stride_con, scale_con, offset, limit, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1607
          if (!conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1608
            conditional_rc = !loop->dominates_backedge(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1609
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1610
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1611
        default:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1612
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1613
          if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1614
            tty->print_cr("missed RCE opportunity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1615
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1616
          continue;             // Unhandled case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1617
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1618
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1619
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1620
      // Kill the eliminated test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1621
      C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1622
      Node *kill_con = _igvn.intcon( 1-flip );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1623
      set_ctrl(kill_con, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1624
      _igvn.hash_delete(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1625
      iff->set_req(1, kill_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1626
      _igvn._worklist.push(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1627
      // Find surviving projection
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1628
      assert(iff->is_If(), "");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1629
      ProjNode* dp = ((IfNode*)iff)->proj_out(1-flip);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1630
      // Find loads off the surviving projection; remove their control edge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1631
      for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1632
        Node* cd = dp->fast_out(i); // Control-dependent node
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1633
        if( cd->is_Load() ) {   // Loads can now float around in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1634
          _igvn.hash_delete(cd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1635
          // Allow the load to float around in the loop, or before it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1636
          // but NOT before the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1637
          cd->set_req(0, ctrl);   // ctrl, not NULL
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1638
          _igvn._worklist.push(cd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1639
          --i;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1640
          --imax;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1641
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1642
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1643
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1644
    } // End of is IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1645
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1646
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1647
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1648
  // Update loop limits
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1649
  if (conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1650
    pre_limit = (stride_con > 0) ? (Node*)new (C,3) MinINode(pre_limit, orig_limit)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1651
                                 : (Node*)new (C,3) MaxINode(pre_limit, orig_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1652
    register_new_node(pre_limit, pre_ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1653
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1654
  _igvn.hash_delete(pre_opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1655
  pre_opaq->set_req(1, pre_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1656
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1657
  // Note:: we are making the main loop limit no longer precise;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1658
  // need to round up based on stride.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1659
  if( stride_con != 1 && stride_con != -1 ) { // Cutout for common case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1660
    // "Standard" round-up logic:  ([main_limit-init+(y-1)]/y)*y+init
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1661
    // Hopefully, compiler will optimize for powers of 2.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1662
    Node *ctrl = get_ctrl(main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1663
    Node *stride = cl->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1664
    Node *init = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1665
    Node *span = new (C, 3) SubINode(main_limit,init);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1666
    register_new_node(span,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1667
    Node *rndup = _igvn.intcon(stride_con + ((stride_con>0)?-1:1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1668
    Node *add = new (C, 3) AddINode(span,rndup);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1669
    register_new_node(add,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1670
    Node *div = new (C, 3) DivINode(0,add,stride);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1671
    register_new_node(div,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1672
    Node *mul = new (C, 3) MulINode(div,stride);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1673
    register_new_node(mul,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1674
    Node *newlim = new (C, 3) AddINode(mul,init);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1675
    register_new_node(newlim,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1676
    main_limit = newlim;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1677
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1678
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1679
  Node *main_cle = cl->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1680
  Node *main_bol = main_cle->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1681
  // Hacking loop bounds; need private copies of exit test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1682
  if( main_bol->outcnt() > 1 ) {// BoolNode shared?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1683
    _igvn.hash_delete(main_cle);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1684
    main_bol = main_bol->clone();// Clone a private BoolNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1685
    register_new_node( main_bol, main_cle->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1686
    main_cle->set_req(1,main_bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1687
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1688
  Node *main_cmp = main_bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1689
  if( main_cmp->outcnt() > 1 ) { // CmpNode shared?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1690
    _igvn.hash_delete(main_bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1691
    main_cmp = main_cmp->clone();// Clone a private CmpNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1692
    register_new_node( main_cmp, main_cle->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1693
    main_bol->set_req(1,main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1694
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1695
  // Hack the now-private loop bounds
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1696
  _igvn.hash_delete(main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1697
  main_cmp->set_req(2, main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1698
  _igvn._worklist.push(main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1699
  // The OpaqueNode is unshared by design
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1700
  _igvn.hash_delete(opqzm);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1701
  assert( opqzm->outcnt() == 1, "cannot hack shared node" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1702
  opqzm->set_req(1,main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1703
  _igvn._worklist.push(opqzm);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1704
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1705
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1706
//------------------------------DCE_loop_body----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1707
// Remove simplistic dead code from loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1708
void IdealLoopTree::DCE_loop_body() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1709
  for( uint i = 0; i < _body.size(); i++ )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1710
    if( _body.at(i)->outcnt() == 0 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1711
      _body.map( i--, _body.pop() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1712
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1713
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1714
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1715
//------------------------------adjust_loop_exit_prob--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1716
// Look for loop-exit tests with the 50/50 (or worse) guesses from the parsing stage.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1717
// Replace with a 1-in-10 exit guess.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1718
void IdealLoopTree::adjust_loop_exit_prob( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1719
  Node *test = tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1720
  while( test != _head ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1721
    uint top = test->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1722
    if( top == Op_IfTrue || top == Op_IfFalse ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1723
      int test_con = ((ProjNode*)test)->_con;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1724
      assert(top == (uint)(test_con? Op_IfTrue: Op_IfFalse), "sanity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1725
      IfNode *iff = test->in(0)->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1726
      if( iff->outcnt() == 2 ) {        // Ignore dead tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1727
        Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1728
        if( bol && bol->req() > 1 && bol->in(1) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1729
            ((bol->in(1)->Opcode() == Op_StorePConditional ) ||
1500
bea9a90f3e8f 6462850: generate biased locking code in C2 ideal graph
kvn
parents: 1433
diff changeset
  1730
             (bol->in(1)->Opcode() == Op_StoreIConditional ) ||
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1731
             (bol->in(1)->Opcode() == Op_StoreLConditional ) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1732
             (bol->in(1)->Opcode() == Op_CompareAndSwapI ) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1733
             (bol->in(1)->Opcode() == Op_CompareAndSwapL ) ||
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 212
diff changeset
  1734
             (bol->in(1)->Opcode() == Op_CompareAndSwapP ) ||
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 212
diff changeset
  1735
             (bol->in(1)->Opcode() == Op_CompareAndSwapN )))
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1736
          return;               // Allocation loops RARELY take backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1737
        // Find the OTHER exit path from the IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1738
        Node* ex = iff->proj_out(1-test_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1739
        float p = iff->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1740
        if( !phase->is_member( this, ex ) && iff->_fcnt == COUNT_UNKNOWN ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1741
          if( top == Op_IfTrue ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1742
            if( p < (PROB_FAIR + PROB_UNLIKELY_MAG(3))) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1743
              iff->_prob = PROB_STATIC_FREQUENT;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1744
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1745
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1746
            if( p > (PROB_FAIR - PROB_UNLIKELY_MAG(3))) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1747
              iff->_prob = PROB_STATIC_INFREQUENT;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1748
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1749
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1750
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1751
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1752
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1753
    test = phase->idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1754
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1755
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1756
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1757
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1758
//------------------------------policy_do_remove_empty_loop--------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1759
// Micro-benchmark spamming.  Policy is to always remove empty loops.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1760
// The 'DO' part is to replace the trip counter with the value it will
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1761
// have on the last iteration.  This will break the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1762
bool IdealLoopTree::policy_do_remove_empty_loop( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1763
  // Minimum size must be empty loop
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1764
  if (_body.size() > 7/*number of nodes in an empty loop*/)
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1765
    return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1766
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1767
  if (!_head->is_CountedLoop())
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1768
    return false;     // Dead loop
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1769
  CountedLoopNode *cl = _head->as_CountedLoop();
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1770
  if (!cl->loopexit())
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1771
    return false; // Malformed loop
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1772
  if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1773
    return false;             // Infinite loop
8870
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1774
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1775
#ifdef ASSERT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1776
  // Ensure only one phi which is the iv.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1777
  Node* iv = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1778
  for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1779
    Node* n = cl->fast_out(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1780
    if (n->Opcode() == Op_Phi) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1781
      assert(iv == NULL, "Too many phis" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1782
      iv = n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1783
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1784
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1785
  assert(iv == cl->phi(), "Wrong phi" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1786
#endif
8870
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1787
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1788
  // main and post loops have explicitly created zero trip guard
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1789
  bool needs_guard = !cl->is_main_loop() && !cl->is_post_loop();
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1790
  if (needs_guard) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1791
    // Check for an obvious zero trip guard.
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  1792
    Node* inctrl = PhaseIdealLoop::skip_loop_predicates(cl->in(LoopNode::EntryControl));
8870
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1793
    if (inctrl->Opcode() == Op_IfTrue) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1794
      // The test should look like just the backedge of a CountedLoop
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1795
      Node* iff = inctrl->in(0);
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1796
      if (iff->is_If()) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1797
        Node* bol = iff->in(1);
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1798
        if (bol->is_Bool() && bol->as_Bool()->_test._test == cl->loopexit()->test_trip()) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1799
          Node* cmp = bol->in(1);
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1800
          if (cmp->is_Cmp() && cmp->in(1) == cl->init_trip() && cmp->in(2) == cl->limit()) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1801
            needs_guard = false;
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1802
          }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1803
        }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1804
      }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1805
    }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1806
  }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1807
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1808
#ifndef PRODUCT
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1809
  if (PrintOpto) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1810
    tty->print("Removing empty loop with%s zero trip guard", needs_guard ? "out" : "");
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1811
    this->dump_head();
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1812
  } else if (TraceLoopOpts) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1813
    tty->print("Empty with%s zero trip guard   ", needs_guard ? "out" : "");
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1814
    this->dump_head();
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1815
  }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1816
#endif
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1817
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1818
  if (needs_guard) {
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1819
    // Peel the loop to ensure there's a zero trip guard
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1820
    Node_List old_new;
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1821
    phase->do_peeling(this, old_new);
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1822
  }
119881dc9d0b 7024475: loop doesn't terminate when compiled
never
parents: 8732
diff changeset
  1823
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1824
  // Replace the phi at loop head with the final value of the last
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1825
  // iteration.  Then the CountedLoopEnd will collapse (backedge never
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1826
  // taken) and all loop-invariant uses of the exit values will be correct.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1827
  Node *phi = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1828
  Node *final = new (phase->C, 3) SubINode( cl->limit(), cl->stride() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1829
  phase->register_new_node(final,cl->in(LoopNode::EntryControl));
5901
c046f8e9c52b 6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents: 5547
diff changeset
  1830
  phase->_igvn.replace_node(phi,final);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1831
  phase->C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1832
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1833
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1834
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1835
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1836
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1837
//------------------------------iteration_split_impl---------------------------
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1838
bool IdealLoopTree::iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1839
  // Check and remove empty loops (spam micro-benchmarks)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1840
  if( policy_do_remove_empty_loop(phase) )
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1841
    return true;  // Here we removed an empty loop
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1842
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1843
  bool should_peel = policy_peeling(phase); // Should we peel?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1844
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1845
  bool should_unswitch = policy_unswitching(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1846
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1847
  // Non-counted loops may be peeled; exactly 1 iteration is peeled.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1848
  // This removes loop-invariant tests (usually null checks).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1849
  if( !_head->is_CountedLoop() ) { // Non-counted loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1850
    if (PartialPeelLoop && phase->partial_peel(this, old_new)) {
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1851
      // Partial peel succeeded so terminate this round of loop opts
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1852
      return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1853
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1854
    if( should_peel ) {            // Should we peel?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1855
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1856
      if (PrintOpto) tty->print_cr("should_peel");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1857
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1858
      phase->do_peeling(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1859
    } else if( should_unswitch ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1860
      phase->do_unswitching(this, old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1861
    }
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1862
    return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1863
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1864
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1865
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1866
  if( !cl->loopexit() ) return true; // Ignore various kinds of broken loops
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1867
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1868
  // Do nothing special to pre- and post- loops
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1869
  if( cl->is_pre_loop() || cl->is_post_loop() ) return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1870
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1871
  // Compute loop trip count from profile data
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1872
  compute_profile_trip_cnt(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1873
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1874
  // Before attempting fancy unrolling, RCE or alignment, see if we want
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1875
  // to completely unroll this loop or do loop unswitching.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1876
  if( cl->is_normal_loop() ) {
2876
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1877
    if (should_unswitch) {
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1878
      phase->do_unswitching(this, old_new);
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1879
      return true;
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1880
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1881
    bool should_maximally_unroll =  policy_maximally_unroll(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1882
    if( should_maximally_unroll ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1883
      // Here we did some unrolling and peeling.  Eventually we will
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1884
      // completely unroll this loop and it will no longer be a loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1885
      phase->do_maximally_unroll(this,old_new);
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1886
      return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1887
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1888
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1889
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1890
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1891
  // Counted loops may be peeled, may need some iterations run up
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1892
  // front for RCE, and may want to align loop refs to a cache
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1893
  // line.  Thus we clone a full loop up front whose trip count is
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1894
  // at least 1 (if peeling), but may be several more.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1895
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1896
  // The main loop will start cache-line aligned with at least 1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1897
  // iteration of the unrolled body (zero-trip test required) and
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1898
  // will have some range checks removed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1899
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1900
  // A post-loop will finish any odd iterations (leftover after
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1901
  // unrolling), plus any needed for RCE purposes.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1902
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1903
  bool should_unroll = policy_unroll(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1904
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1905
  bool should_rce = policy_range_check(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1906
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1907
  bool should_align = policy_align(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1908
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1909
  // If not RCE'ing (iteration splitting) or Aligning, then we do not
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1910
  // need a pre-loop.  We may still need to peel an initial iteration but
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1911
  // we will not be needing an unknown number of pre-iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1912
  //
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1913
  // Basically, if may_rce_align reports FALSE first time through,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1914
  // we will not be able to later do RCE or Aligning on this loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1915
  bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1916
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1917
  // If we have any of these conditions (RCE, alignment, unrolling) met, then
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1918
  // we switch to the pre-/main-/post-loop model.  This model also covers
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1919
  // peeling.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1920
  if( should_rce || should_align || should_unroll ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1921
    if( cl->is_normal_loop() )  // Convert to 'pre/main/post' loops
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1922
      phase->insert_pre_post_loops(this,old_new, !may_rce_align);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1923
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1924
    // Adjust the pre- and main-loop limits to let the pre and post loops run
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1925
    // with full checks, but the main-loop with no checks.  Remove said
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1926
    // checks from the main body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1927
    if( should_rce )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1928
      phase->do_range_check(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1929
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1930
    // Double loop body for unrolling.  Adjust the minimum-trip test (will do
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1931
    // twice as many iterations as before) and the main body limit (only do
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1932
    // an even number of trips).  If we are peeling, we might enable some RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1933
    // and we'd rather unroll the post-RCE'd loop SO... do not unroll if
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1934
    // peeling.
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1935
      if( should_unroll && !should_peel )
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1936
        phase->do_unroll(this,old_new, true);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1937
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1938
    // Adjust the pre-loop limits to align the main body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1939
    // iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1940
    if( should_align )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1941
      Unimplemented();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1942
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1943
  } else {                      // Else we have an unchanged counted loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1944
    if( should_peel )           // Might want to peel but do nothing else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1945
      phase->do_peeling(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1946
  }
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1947
  return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1948
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1949
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1950
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1951
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1952
//------------------------------iteration_split--------------------------------
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1953
bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1954
  // Recursively iteration split nested loops
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1955
  if (_child && !_child->iteration_split(phase, old_new))
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1956
    return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1957
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1958
  // Clean out prior deadwood
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1959
  DCE_loop_body();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1960
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1961
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1962
  // Look for loop-exit tests with my 50/50 guesses from the Parsing stage.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1963
  // Replace with a 1-in-10 exit guess.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1964
  if (_parent /*not the root loop*/ &&
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1965
      !_irreducible &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1966
      // Also ignore the occasional dead backedge
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1967
      !tail()->is_top()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1968
    adjust_loop_exit_prob(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1969
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1970
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1971
  // Gate unrolling, RCE and peeling efforts.
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1972
  if (!_child &&                // If not an inner loop, do not split
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1973
      !_irreducible &&
212
cd4963e67949 6667612: (Escape Analysis) disable loop cloning if it has a scalar replaceable allocation
kvn
parents: 1
diff changeset
  1974
      _allow_optimizations &&
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1975
      !tail()->is_top()) {     // Also ignore the occasional dead backedge
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1976
    if (!_has_call) {
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1977
        if (!iteration_split_impl(phase, old_new)) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1978
          return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1979
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1980
    } else if (policy_unswitching(phase)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1981
      phase->do_unswitching(this, old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1982
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1983
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1984
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1985
  // Minor offset re-organization to remove loop-fallout uses of
8732
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1986
  // trip counter when there was no major reshaping.
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1987
  phase->reorg_offsets(this);
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1988
16fc1c68714b 7008866: Missing loop predicate for loop with multiple entries
kvn
parents: 7397
diff changeset
  1989
  if (_next && !_next->iteration_split(phase, old_new))
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1990
    return false;
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1991
  return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1992
}
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1993
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1994
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  1995
//=============================================================================
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  1996
// Process all the loops in the loop tree and replace any fill
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  1997
// patterns with an intrisc version.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  1998
bool PhaseIdealLoop::do_intrinsify_fill() {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  1999
  bool changed = false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2000
  for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2001
    IdealLoopTree* lpt = iter.current();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2002
    changed |= intrinsify_fill(lpt);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2003
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2004
  return changed;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2005
}
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2006
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2007
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2008
// Examine an inner loop looking for a a single store of an invariant
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2009
// value in a unit stride loop,
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2010
bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2011
                                     Node*& shift, Node*& con) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2012
  const char* msg = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2013
  Node* msg_node = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2014
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2015
  store_value = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2016
  con = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2017
  shift = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2018
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2019
  // Process the loop looking for stores.  If there are multiple
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2020
  // stores or extra control flow give at this point.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2021
  CountedLoopNode* head = lpt->_head->as_CountedLoop();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2022
  for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2023
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2024
    if (n->outcnt() == 0) continue; // Ignore dead
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2025
    if (n->is_Store()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2026
      if (store != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2027
        msg = "multiple stores";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2028
        break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2029
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2030
      int opc = n->Opcode();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2031
      if (opc == Op_StoreP || opc == Op_StoreN || opc == Op_StoreCM) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2032
        msg = "oop fills not handled";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2033
        break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2034
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2035
      Node* value = n->in(MemNode::ValueIn);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2036
      if (!lpt->is_invariant(value)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2037
        msg  = "variant store value";
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2038
      } else if (!_igvn.type(n->in(MemNode::Address))->isa_aryptr()) {
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2039
        msg = "not array address";
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2040
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2041
      store = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2042
      store_value = value;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2043
    } else if (n->is_If() && n != head->loopexit()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2044
      msg = "extra control flow";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2045
      msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2046
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2047
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2048
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2049
  if (store == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2050
    // No store in loop
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2051
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2052
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2053
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2054
  if (msg == NULL && head->stride_con() != 1) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2055
    // could handle negative strides too
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2056
    if (head->stride_con() < 0) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2057
      msg = "negative stride";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2058
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2059
      msg = "non-unit stride";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2060
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2061
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2062
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2063
  if (msg == NULL && !store->in(MemNode::Address)->is_AddP()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2064
    msg = "can't handle store address";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2065
    msg_node = store->in(MemNode::Address);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2066
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2067
6739
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2068
  if (msg == NULL &&
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2069
      (!store->in(MemNode::Memory)->is_Phi() ||
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2070
       store->in(MemNode::Memory)->in(LoopNode::LoopBackControl) != store)) {
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2071
    msg = "store memory isn't proper phi";
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2072
    msg_node = store->in(MemNode::Memory);
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2073
  }
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2074
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2075
  // Make sure there is an appropriate fill routine
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2076
  BasicType t = store->as_Mem()->memory_type();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2077
  const char* fill_name;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2078
  if (msg == NULL &&
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2079
      StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2080
    msg = "unsupported store";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2081
    msg_node = store;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2082
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2083
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2084
  if (msg != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2085
#ifndef PRODUCT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2086
    if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2087
      tty->print_cr("not fill intrinsic candidate: %s", msg);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2088
      if (msg_node != NULL) msg_node->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2089
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2090
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2091
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2092
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2093
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2094
  // Make sure the address expression can be handled.  It should be
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2095
  // head->phi * elsize + con.  head->phi might have a ConvI2L.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2096
  Node* elements[4];
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2097
  Node* conv = NULL;
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2098
  bool found_index = false;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2099
  int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2100
  for (int e = 0; e < count; e++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2101
    Node* n = elements[e];
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2102
    if (n->is_Con() && con == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2103
      con = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2104
    } else if (n->Opcode() == Op_LShiftX && shift == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2105
      Node* value = n->in(1);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2106
#ifdef _LP64
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2107
      if (value->Opcode() == Op_ConvI2L) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2108
        conv = value;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2109
        value = value->in(1);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2110
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2111
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2112
      if (value != head->phi()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2113
        msg = "unhandled shift in address";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2114
      } else {
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2115
        found_index = true;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2116
        shift = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2117
        assert(type2aelembytes(store->as_Mem()->memory_type(), true) == 1 << shift->in(2)->get_int(), "scale should match");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2118
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2119
    } else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2120
      if (n->in(1) == head->phi()) {
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2121
        found_index = true;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2122
        conv = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2123
      } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2124
        msg = "unhandled input to ConvI2L";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2125
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2126
    } else if (n == head->phi()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2127
      // no shift, check below for allowed cases
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2128
      found_index = true;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2129
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2130
      msg = "unhandled node in address";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2131
      msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2132
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2133
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2134
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2135
  if (count == -1) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2136
    msg = "malformed address expression";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2137
    msg_node = store;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2138
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2139
6455
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2140
  if (!found_index) {
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2141
    msg = "missing use of index";
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2142
  }
4a8142e5f75f 6982533: Crash in ~StubRoutines::jbyte_fill with AggressiveOpts enabled
never
parents: 6433
diff changeset
  2143
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2144
  // byte sized items won't have a shift
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2145
  if (msg == NULL && shift == NULL && t != T_BYTE && t != T_BOOLEAN) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2146
    msg = "can't find shift";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2147
    msg_node = store;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2148
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2149
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2150
  if (msg != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2151
#ifndef PRODUCT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2152
    if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2153
      tty->print_cr("not fill intrinsic: %s", msg);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2154
      if (msg_node != NULL) msg_node->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2155
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2156
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2157
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2158
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2159
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2160
  // No make sure all the other nodes in the loop can be handled
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2161
  VectorSet ok(Thread::current()->resource_area());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2162
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2163
  // store related values are ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2164
  ok.set(store->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2165
  ok.set(store->in(MemNode::Memory)->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2166
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2167
  // Loop structure is ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2168
  ok.set(head->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2169
  ok.set(head->loopexit()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2170
  ok.set(head->phi()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2171
  ok.set(head->incr()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2172
  ok.set(head->loopexit()->cmp_node()->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2173
  ok.set(head->loopexit()->in(1)->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2174
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2175
  // Address elements are ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2176
  if (con)   ok.set(con->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2177
  if (shift) ok.set(shift->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2178
  if (conv)  ok.set(conv->_idx);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2179
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2180
  for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2181
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2182
    if (n->outcnt() == 0) continue; // Ignore dead
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2183
    if (ok.test(n->_idx)) continue;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2184
    // Backedge projection is ok
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2185
    if (n->is_IfTrue() && n->in(0) == head->loopexit()) continue;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2186
    if (!n->is_AddP()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2187
      msg = "unhandled node";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2188
      msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2189
      break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2190
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2191
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2192
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2193
  // Make sure no unexpected values are used outside the loop
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2194
  for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2195
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2196
    // These values can be replaced with other nodes if they are used
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2197
    // outside the loop.
6739
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2198
    if (n == store || n == head->loopexit() || n == head->incr() || n == store->in(MemNode::Memory)) continue;
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2199
    for (SimpleDUIterator iter(n); iter.has_next(); iter.next()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2200
      Node* use = iter.get();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2201
      if (!lpt->_body.contains(use)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2202
        msg = "node is used outside loop";
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2203
        // lpt->_body.dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2204
        msg_node = n;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2205
        break;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2206
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2207
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2208
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2209
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2210
#ifdef ASSERT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2211
  if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2212
    if (msg != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2213
      tty->print_cr("no fill intrinsic: %s", msg);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2214
      if (msg_node != NULL) msg_node->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2215
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2216
      tty->print_cr("fill intrinsic for:");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2217
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2218
    store->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2219
    if (Verbose) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2220
      lpt->_body.dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2221
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2222
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2223
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2224
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2225
  return msg == NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2226
}
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2227
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2228
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2229
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2230
bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2231
  // Only for counted inner loops
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2232
  if (!lpt->is_counted() || !lpt->is_inner()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2233
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2234
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2235
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2236
  // Must have constant stride
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2237
  CountedLoopNode* head = lpt->_head->as_CountedLoop();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2238
  if (!head->stride_is_con() || !head->is_normal_loop()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2239
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2240
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2241
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2242
  // Check that the body only contains a store of a loop invariant
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2243
  // value that is indexed by the loop phi.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2244
  Node* store = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2245
  Node* store_value = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2246
  Node* shift = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2247
  Node* offset = NULL;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2248
  if (!match_fill_loop(lpt, store, store_value, shift, offset)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2249
    return false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2250
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2251
9101
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2252
#ifndef PRODUCT
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2253
  if (TraceLoopOpts) {
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2254
    tty->print("ArrayFill    ");
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2255
    lpt->dump_head();
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2256
  }
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2257
#endif
ff58f9a8e31c 7004535: Clone loop predicate during loop unswitch
kvn
parents: 8884
diff changeset
  2258
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2259
  // Now replace the whole loop body by a call to a fill routine that
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2260
  // covers the same region as the loop.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2261
  Node* base = store->in(MemNode::Address)->as_AddP()->in(AddPNode::Base);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2262
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2263
  // Build an expression for the beginning of the copy region
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2264
  Node* index = head->init_trip();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2265
#ifdef _LP64
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2266
  index = new (C, 2) ConvI2LNode(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2267
  _igvn.register_new_node_with_optimizer(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2268
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2269
  if (shift != NULL) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2270
    // byte arrays don't require a shift but others do.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2271
    index = new (C, 3) LShiftXNode(index, shift->in(2));
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2272
    _igvn.register_new_node_with_optimizer(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2273
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2274
  index = new (C, 4) AddPNode(base, base, index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2275
  _igvn.register_new_node_with_optimizer(index);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2276
  Node* from = new (C, 4) AddPNode(base, index, offset);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2277
  _igvn.register_new_node_with_optimizer(from);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2278
  // Compute the number of elements to copy
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2279
  Node* len = new (C, 3) SubINode(head->limit(), head->init_trip());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2280
  _igvn.register_new_node_with_optimizer(len);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2281
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2282
  BasicType t = store->as_Mem()->memory_type();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2283
  bool aligned = false;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2284
  if (offset != NULL && head->init_trip()->is_Con()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2285
    int element_size = type2aelembytes(t);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2286
    aligned = (offset->find_intptr_t_type()->get_con() + head->init_trip()->get_int() * element_size) % HeapWordSize == 0;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2287
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2288
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2289
  // Build a call to the fill routine
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2290
  const char* fill_name;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2291
  address fill = StubRoutines::select_fill_function(t, aligned, fill_name);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2292
  assert(fill != NULL, "what?");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2293
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2294
  // Convert float/double to int/long for fill routines
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2295
  if (t == T_FLOAT) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2296
    store_value = new (C, 2) MoveF2INode(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2297
    _igvn.register_new_node_with_optimizer(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2298
  } else if (t == T_DOUBLE) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2299
    store_value = new (C, 2) MoveD2LNode(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2300
    _igvn.register_new_node_with_optimizer(store_value);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2301
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2302
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2303
  Node* mem_phi = store->in(MemNode::Memory);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2304
  Node* result_ctrl;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2305
  Node* result_mem;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2306
  const TypeFunc* call_type = OptoRuntime::array_fill_Type();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2307
  int size = call_type->domain()->cnt();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2308
  CallLeafNode *call = new (C, size) CallLeafNoFPNode(call_type, fill,
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2309
                                                      fill_name, TypeAryPtr::get_array_body_type(t));
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2310
  call->init_req(TypeFunc::Parms+0, from);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2311
  call->init_req(TypeFunc::Parms+1, store_value);
6770
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2312
#ifdef _LP64
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2313
  len = new (C, 2) ConvI2LNode(len);
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2314
  _igvn.register_new_node_with_optimizer(len);
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2315
#endif
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2316
  call->init_req(TypeFunc::Parms+2, len);
6770
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2317
#ifdef _LP64
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2318
  call->init_req(TypeFunc::Parms+3, C->top());
a67870aaedb3 6980792: Crash "exception happened outside interpreter, nmethods and vtable stubs (1)"
never
parents: 6739
diff changeset
  2319
#endif
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2320
  call->init_req( TypeFunc::Control, head->init_control());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2321
  call->init_req( TypeFunc::I_O    , C->top() )        ;   // does no i/o
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2322
  call->init_req( TypeFunc::Memory ,  mem_phi->in(LoopNode::EntryControl) );
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2323
  call->init_req( TypeFunc::ReturnAdr, C->start()->proj_out(TypeFunc::ReturnAdr) );
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2324
  call->init_req( TypeFunc::FramePtr, C->start()->proj_out(TypeFunc::FramePtr) );
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2325
  _igvn.register_new_node_with_optimizer(call);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2326
  result_ctrl = new (C, 1) ProjNode(call,TypeFunc::Control);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2327
  _igvn.register_new_node_with_optimizer(result_ctrl);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2328
  result_mem = new (C, 1) ProjNode(call,TypeFunc::Memory);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2329
  _igvn.register_new_node_with_optimizer(result_mem);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2330
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2331
  // If this fill is tightly coupled to an allocation and overwrites
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2332
  // the whole body, allow it to take over the zeroing.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2333
  AllocateNode* alloc = AllocateNode::Ideal_allocation(base, this);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2334
  if (alloc != NULL && alloc->is_AllocateArray()) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2335
    Node* length = alloc->as_AllocateArray()->Ideal_length();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2336
    if (head->limit() == length &&
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2337
        head->init_trip() == _igvn.intcon(0)) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2338
      if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2339
        tty->print_cr("Eliminated zeroing in allocation");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2340
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2341
      alloc->maybe_set_complete(&_igvn);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2342
    } else {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2343
#ifdef ASSERT
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2344
      if (TraceOptimizeFill) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2345
        tty->print_cr("filling array but bounds don't match");
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2346
        alloc->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2347
        head->init_trip()->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2348
        head->limit()->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2349
        length->dump();
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2350
      }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2351
#endif
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2352
    }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2353
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2354
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2355
  // Redirect the old control and memory edges that are outside the loop.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2356
  Node* exit = head->loopexit()->proj_out(0);
6739
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2357
  // Sometimes the memory phi of the head is used as the outgoing
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2358
  // state of the loop.  It's safe in this case to replace it with the
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2359
  // result_mem.
750998abc20f 6984979: OptimizeFill misses some cases with an odd memory graph
never
parents: 6455
diff changeset
  2360
  _igvn.replace_node(store->in(MemNode::Memory), result_mem);
6433
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2361
  _igvn.replace_node(exit, result_ctrl);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2362
  _igvn.replace_node(store, result_mem);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2363
  // Any uses the increment outside of the loop become the loop limit.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2364
  _igvn.replace_node(head->incr(), head->limit());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2365
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2366
  // Disconnect the head from the loop.
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2367
  for (uint i = 0; i < lpt->_body.size(); i++) {
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2368
    Node* n = lpt->_body.at(i);
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2369
    _igvn.replace_node(n, C->top());
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2370
  }
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2371
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2372
  return true;
b0e4fafdc38b 4809552: Optimize Arrays.fill(...)
never
parents: 5901
diff changeset
  2373
}