hotspot/src/share/vm/opto/loopTransform.cpp
author kvn
Mon, 28 Jun 2010 14:54:39 -0700
changeset 5901 c046f8e9c52b
parent 5547 f4b087cbb361
child 6433 b0e4fafdc38b
permissions -rw-r--r--
6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist() Summary: Use replace_node() method instead of subsume_node(). Reviewed-by: jrose, never
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     1
/*
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
     2
 * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     4
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
489c9b5090e2 Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
489c9b5090e2 Initial load
duke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     8
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
489c9b5090e2 Initial load
duke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
489c9b5090e2 Initial load
duke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
489c9b5090e2 Initial load
duke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
489c9b5090e2 Initial load
duke
parents:
diff changeset
    13
 * accompanied this code).
489c9b5090e2 Initial load
duke
parents:
diff changeset
    14
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
489c9b5090e2 Initial load
duke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
489c9b5090e2 Initial load
duke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    18
 *
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5054
diff changeset
    21
 * questions.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    22
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    23
 */
489c9b5090e2 Initial load
duke
parents:
diff changeset
    24
489c9b5090e2 Initial load
duke
parents:
diff changeset
    25
#include "incls/_precompiled.incl"
489c9b5090e2 Initial load
duke
parents:
diff changeset
    26
#include "incls/_loopTransform.cpp.incl"
489c9b5090e2 Initial load
duke
parents:
diff changeset
    27
489c9b5090e2 Initial load
duke
parents:
diff changeset
    28
//------------------------------is_loop_exit-----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    29
// Given an IfNode, return the loop-exiting projection or NULL if both
489c9b5090e2 Initial load
duke
parents:
diff changeset
    30
// arms remain in the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    31
Node *IdealLoopTree::is_loop_exit(Node *iff) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    32
  if( iff->outcnt() != 2 ) return NULL; // Ignore partially dead tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
    33
  PhaseIdealLoop *phase = _phase;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    34
  // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
    35
  // we need loop unswitching instead of peeling.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    36
  if( !is_member(phase->get_loop( iff->raw_out(0) )) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
    37
    return iff->raw_out(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    38
  if( !is_member(phase->get_loop( iff->raw_out(1) )) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
    39
    return iff->raw_out(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    40
  return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    41
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    42
489c9b5090e2 Initial load
duke
parents:
diff changeset
    43
489c9b5090e2 Initial load
duke
parents:
diff changeset
    44
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
    45
489c9b5090e2 Initial load
duke
parents:
diff changeset
    46
489c9b5090e2 Initial load
duke
parents:
diff changeset
    47
//------------------------------record_for_igvn----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    48
// Put loop body on igvn work list
489c9b5090e2 Initial load
duke
parents:
diff changeset
    49
void IdealLoopTree::record_for_igvn() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    50
  for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    51
    Node *n = _body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    52
    _phase->_igvn._worklist.push(n);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    53
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    54
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    55
489c9b5090e2 Initial load
duke
parents:
diff changeset
    56
//------------------------------compute_profile_trip_cnt----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
    57
// Compute loop trip count from profile data as
489c9b5090e2 Initial load
duke
parents:
diff changeset
    58
//    (backedge_count + loop_exit_count) / loop_exit_count
489c9b5090e2 Initial load
duke
parents:
diff changeset
    59
void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    60
  if (!_head->is_CountedLoop()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    61
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    62
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    63
  CountedLoopNode* head = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    64
  if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    65
    return; // Already computed
489c9b5090e2 Initial load
duke
parents:
diff changeset
    66
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    67
  float trip_cnt = (float)max_jint; // default is big
489c9b5090e2 Initial load
duke
parents:
diff changeset
    68
489c9b5090e2 Initial load
duke
parents:
diff changeset
    69
  Node* back = head->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    70
  while (back != head) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    71
    if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    72
        back->in(0) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    73
        back->in(0)->is_If() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    74
        back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    75
        back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    76
      break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    77
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    78
    back = phase->idom(back);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    79
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    80
  if (back != head) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    81
    assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
    82
           back->in(0), "if-projection exists");
489c9b5090e2 Initial load
duke
parents:
diff changeset
    83
    IfNode* back_if = back->in(0)->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    84
    float loop_back_cnt = back_if->_fcnt * back_if->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    85
489c9b5090e2 Initial load
duke
parents:
diff changeset
    86
    // Now compute a loop exit count
489c9b5090e2 Initial load
duke
parents:
diff changeset
    87
    float loop_exit_cnt = 0.0f;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    88
    for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    89
      Node *n = _body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
    90
      if( n->is_If() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    91
        IfNode *iff = n->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    92
        if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    93
          Node *exit = is_loop_exit(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    94
          if( exit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    95
            float exit_prob = iff->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    96
            if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    97
            if (exit_prob > PROB_MIN) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    98
              float exit_cnt = iff->_fcnt * exit_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    99
              loop_exit_cnt += exit_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   100
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   101
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   102
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   103
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   104
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   105
    if (loop_exit_cnt > 0.0f) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   106
      trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   107
    } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   108
      // No exit count so use
489c9b5090e2 Initial load
duke
parents:
diff changeset
   109
      trip_cnt = loop_back_cnt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   110
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   111
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   112
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   113
  if (TraceProfileTripCount) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   114
    tty->print_cr("compute_profile_trip_cnt  lp: %d cnt: %f\n", head->_idx, trip_cnt);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   115
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   116
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   117
  head->set_profile_trip_cnt(trip_cnt);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   118
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   119
489c9b5090e2 Initial load
duke
parents:
diff changeset
   120
//---------------------is_invariant_addition-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   121
// Return nonzero index of invariant operand for an Add or Sub
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1500
diff changeset
   122
// of (nonconstant) invariant and variant values. Helper for reassociate_invariants.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   123
int IdealLoopTree::is_invariant_addition(Node* n, PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   124
  int op = n->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   125
  if (op == Op_AddI || op == Op_SubI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   126
    bool in1_invar = this->is_invariant(n->in(1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   127
    bool in2_invar = this->is_invariant(n->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   128
    if (in1_invar && !in2_invar) return 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   129
    if (!in1_invar && in2_invar) return 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   130
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   131
  return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   132
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   133
489c9b5090e2 Initial load
duke
parents:
diff changeset
   134
//---------------------reassociate_add_sub-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   135
// Reassociate invariant add and subtract expressions:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   136
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
   137
// inv1 + (x + inv2)  =>  ( inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   138
// (x + inv2) + inv1  =>  ( inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   139
// inv1 + (x - inv2)  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   140
// inv1 - (inv2 - x)  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   141
// (x + inv2) - inv1  =>  (-inv1 + inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   142
// (x - inv2) + inv1  =>  ( inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   143
// (x - inv2) - inv1  =>  (-inv1 - inv2) + x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   144
// inv1 + (inv2 - x)  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   145
// inv1 - (x - inv2)  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   146
// (inv2 - x) + inv1  =>  ( inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   147
// (inv2 - x) - inv1  =>  (-inv1 + inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   148
// inv1 - (x + inv2)  =>  ( inv1 - inv2) - x
489c9b5090e2 Initial load
duke
parents:
diff changeset
   149
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
   150
Node* IdealLoopTree::reassociate_add_sub(Node* n1, PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   151
  if (!n1->is_Add() && !n1->is_Sub() || n1->outcnt() == 0) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   152
  if (is_invariant(n1)) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   153
  int inv1_idx = is_invariant_addition(n1, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   154
  if (!inv1_idx) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   155
  // Don't mess with add of constant (igvn moves them to expression tree root.)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   156
  if (n1->is_Add() && n1->in(2)->is_Con()) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   157
  Node* inv1 = n1->in(inv1_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   158
  Node* n2 = n1->in(3 - inv1_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   159
  int inv2_idx = is_invariant_addition(n2, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   160
  if (!inv2_idx) return NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   161
  Node* x    = n2->in(3 - inv2_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   162
  Node* inv2 = n2->in(inv2_idx);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   163
489c9b5090e2 Initial load
duke
parents:
diff changeset
   164
  bool neg_x    = n2->is_Sub() && inv2_idx == 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   165
  bool neg_inv2 = n2->is_Sub() && inv2_idx == 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   166
  bool neg_inv1 = n1->is_Sub() && inv1_idx == 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   167
  if (n1->is_Sub() && inv1_idx == 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   168
    neg_x    = !neg_x;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   169
    neg_inv2 = !neg_inv2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   170
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   171
  Node* inv1_c = phase->get_ctrl(inv1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   172
  Node* inv2_c = phase->get_ctrl(inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   173
  Node* n_inv1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   174
  if (neg_inv1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   175
    Node *zero = phase->_igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   176
    phase->set_ctrl(zero, phase->C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   177
    n_inv1 = new (phase->C, 3) SubINode(zero, inv1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   178
    phase->register_new_node(n_inv1, inv1_c);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   179
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   180
    n_inv1 = inv1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   181
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   182
  Node* inv;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   183
  if (neg_inv2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   184
    inv = new (phase->C, 3) SubINode(n_inv1, inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   185
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   186
    inv = new (phase->C, 3) AddINode(n_inv1, inv2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   187
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   188
  phase->register_new_node(inv, phase->get_early_ctrl(inv));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   189
489c9b5090e2 Initial load
duke
parents:
diff changeset
   190
  Node* addx;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   191
  if (neg_x) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   192
    addx = new (phase->C, 3) SubINode(inv, x);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   193
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   194
    addx = new (phase->C, 3) AddINode(x, inv);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   195
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   196
  phase->register_new_node(addx, phase->get_ctrl(x));
5901
c046f8e9c52b 6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents: 5547
diff changeset
   197
  phase->_igvn.replace_node(n1, addx);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   198
  return addx;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   199
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   200
489c9b5090e2 Initial load
duke
parents:
diff changeset
   201
//---------------------reassociate_invariants-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   202
// Reassociate invariant expressions:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   203
void IdealLoopTree::reassociate_invariants(PhaseIdealLoop *phase) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   204
  for (int i = _body.size() - 1; i >= 0; i--) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   205
    Node *n = _body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   206
    for (int j = 0; j < 5; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   207
      Node* nn = reassociate_add_sub(n, phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   208
      if (nn == NULL) break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   209
      n = nn; // again
489c9b5090e2 Initial load
duke
parents:
diff changeset
   210
    };
489c9b5090e2 Initial load
duke
parents:
diff changeset
   211
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   212
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   213
489c9b5090e2 Initial load
duke
parents:
diff changeset
   214
//------------------------------policy_peeling---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   215
// Return TRUE or FALSE if the loop should be peeled or not.  Peel if we can
489c9b5090e2 Initial load
duke
parents:
diff changeset
   216
// make some loop-invariant test (usually a null-check) happen before the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   217
bool IdealLoopTree::policy_peeling( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   218
  Node *test = ((IdealLoopTree*)this)->tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   219
  int  body_size = ((IdealLoopTree*)this)->_body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   220
  int  uniq      = phase->C->unique();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   221
  // Peeling does loop cloning which can result in O(N^2) node construction
489c9b5090e2 Initial load
duke
parents:
diff changeset
   222
  if( body_size > 255 /* Prevent overflow for large body_size */
489c9b5090e2 Initial load
duke
parents:
diff changeset
   223
      || (body_size * body_size + uniq > MaxNodeLimit) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   224
    return false;           // too large to safely clone
489c9b5090e2 Initial load
duke
parents:
diff changeset
   225
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   226
  while( test != _head ) {      // Scan till run off top of loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   227
    if( test->is_If() ) {       // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   228
      Node *ctrl = phase->get_ctrl(test->in(1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   229
      if (ctrl->is_top())
489c9b5090e2 Initial load
duke
parents:
diff changeset
   230
        return false;           // Found dead test on live IF?  No peeling!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   231
      // Standard IF only has one input value to check for loop invariance
489c9b5090e2 Initial load
duke
parents:
diff changeset
   232
      assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   233
      // Condition is not a member of this loop?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   234
      if( !is_member(phase->get_loop(ctrl)) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   235
          is_loop_exit(test) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   236
        return true;            // Found reason to peel!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   237
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   238
    // Walk up dominators to loop _head looking for test which is
489c9b5090e2 Initial load
duke
parents:
diff changeset
   239
    // executed on every path thru loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   240
    test = phase->idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   241
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   242
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   243
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   244
489c9b5090e2 Initial load
duke
parents:
diff changeset
   245
//------------------------------peeled_dom_test_elim---------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   246
// If we got the effect of peeling, either by actually peeling or by making
489c9b5090e2 Initial load
duke
parents:
diff changeset
   247
// a pre-loop which must execute at least once, we can remove all
489c9b5090e2 Initial load
duke
parents:
diff changeset
   248
// loop-invariant dominated tests in the main body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   249
void PhaseIdealLoop::peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   250
  bool progress = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   251
  while( progress ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   252
    progress = false;           // Reset for next iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
   253
    Node *prev = loop->_head->in(LoopNode::LoopBackControl);//loop->tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   254
    Node *test = prev->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   255
    while( test != loop->_head ) { // Scan till run off top of loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   256
489c9b5090e2 Initial load
duke
parents:
diff changeset
   257
      int p_op = prev->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   258
      if( (p_op == Op_IfFalse || p_op == Op_IfTrue) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   259
          test->is_If() &&      // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   260
          !test->in(1)->is_Con() && // And not already obvious?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   261
          // Condition is not a member of this loop?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   262
          !loop->is_member(get_loop(get_ctrl(test->in(1))))){
489c9b5090e2 Initial load
duke
parents:
diff changeset
   263
        // Walk loop body looking for instances of this test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   264
        for( uint i = 0; i < loop->_body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   265
          Node *n = loop->_body.at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   266
          if( n->is_If() && n->in(1) == test->in(1) /*&& n != loop->tail()->in(0)*/ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   267
            // IfNode was dominated by version in peeled loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
   268
            progress = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   269
            dominated_by( old_new[prev->_idx], n );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   270
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   271
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   272
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   273
      prev = test;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   274
      test = idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   275
    } // End of scan tests in loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   276
489c9b5090e2 Initial load
duke
parents:
diff changeset
   277
  } // End of while( progress )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   278
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   279
489c9b5090e2 Initial load
duke
parents:
diff changeset
   280
//------------------------------do_peeling-------------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   281
// Peel the first iteration of the given loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   282
// Step 1: Clone the loop body.  The clone becomes the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   283
//         The pre-loop illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   284
// Step 2: Make the old-loop fall-in edges point to the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   285
//         Do this by making the old-loop fall-in edges act as if they came
489c9b5090e2 Initial load
duke
parents:
diff changeset
   286
//         around the loopback from the prior iteration (follow the old-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   287
//         backedges) and then map to the new peeled iteration.  This leaves
489c9b5090e2 Initial load
duke
parents:
diff changeset
   288
//         the pre-loop with only 1 user (the new peeled iteration), but the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   289
//         peeled-loop backedge has 2 users.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   290
// Step 3: Cut the backedge on the clone (so its not a loop) and remove the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   291
//         extra backedge user.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   292
void PhaseIdealLoop::do_peeling( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   293
489c9b5090e2 Initial load
duke
parents:
diff changeset
   294
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   295
  // Peeling a 'main' loop in a pre/main/post situation obfuscates the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   296
  // 'pre' loop from the main and the 'pre' can no longer have it's
489c9b5090e2 Initial load
duke
parents:
diff changeset
   297
  // iterations adjusted.  Therefore, we need to declare this loop as
489c9b5090e2 Initial load
duke
parents:
diff changeset
   298
  // no longer a 'main' loop; it will need new pre and post loops before
489c9b5090e2 Initial load
duke
parents:
diff changeset
   299
  // we can do further RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   300
  Node *h = loop->_head;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   301
  if( h->is_CountedLoop() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   302
    CountedLoopNode *cl = h->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   303
    assert(cl->trip_count() > 0, "peeling a fully unrolled loop");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   304
    cl->set_trip_count(cl->trip_count() - 1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   305
    if( cl->is_main_loop() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   306
      cl->set_normal_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   307
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   308
      if( PrintOpto && VerifyLoopOptimizations ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   309
        tty->print("Peeling a 'main' loop; resetting to 'normal' ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   310
        loop->dump_head();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   311
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   312
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   313
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   314
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   315
489c9b5090e2 Initial load
duke
parents:
diff changeset
   316
  // Step 1: Clone the loop body.  The clone becomes the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   317
  //         The pre-loop illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   318
  clone_loop( loop, old_new, dom_depth(loop->_head) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   319
489c9b5090e2 Initial load
duke
parents:
diff changeset
   320
489c9b5090e2 Initial load
duke
parents:
diff changeset
   321
  // Step 2: Make the old-loop fall-in edges point to the peeled iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   322
  //         Do this by making the old-loop fall-in edges act as if they came
489c9b5090e2 Initial load
duke
parents:
diff changeset
   323
  //         around the loopback from the prior iteration (follow the old-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   324
  //         backedges) and then map to the new peeled iteration.  This leaves
489c9b5090e2 Initial load
duke
parents:
diff changeset
   325
  //         the pre-loop with only 1 user (the new peeled iteration), but the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   326
  //         peeled-loop backedge has 2 users.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   327
  for (DUIterator_Fast jmax, j = loop->_head->fast_outs(jmax); j < jmax; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   328
    Node* old = loop->_head->fast_out(j);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   329
    if( old->in(0) == loop->_head && old->req() == 3 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   330
        (old->is_Loop() || old->is_Phi()) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   331
      Node *new_exit_value = old_new[old->in(LoopNode::LoopBackControl)->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   332
      if( !new_exit_value )     // Backedge value is ALSO loop invariant?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   333
        // Then loop body backedge value remains the same.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   334
        new_exit_value = old->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   335
      _igvn.hash_delete(old);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   336
      old->set_req(LoopNode::EntryControl, new_exit_value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   337
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   338
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   339
489c9b5090e2 Initial load
duke
parents:
diff changeset
   340
489c9b5090e2 Initial load
duke
parents:
diff changeset
   341
  // Step 3: Cut the backedge on the clone (so its not a loop) and remove the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   342
  //         extra backedge user.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   343
  Node *nnn = old_new[loop->_head->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   344
  _igvn.hash_delete(nnn);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   345
  nnn->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   346
  for (DUIterator_Fast j2max, j2 = nnn->fast_outs(j2max); j2 < j2max; j2++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   347
    Node* use = nnn->fast_out(j2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   348
    if( use->in(0) == nnn && use->req() == 3 && use->is_Phi() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   349
      _igvn.hash_delete(use);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   350
      use->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   351
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   352
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   353
489c9b5090e2 Initial load
duke
parents:
diff changeset
   354
489c9b5090e2 Initial load
duke
parents:
diff changeset
   355
  // Step 4: Correct dom-depth info.  Set to loop-head depth.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   356
  int dd = dom_depth(loop->_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   357
  set_idom(loop->_head, loop->_head->in(1), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   358
  for (uint j3 = 0; j3 < loop->_body.size(); j3++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   359
    Node *old = loop->_body.at(j3);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   360
    Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   361
    if (!has_ctrl(nnn))
489c9b5090e2 Initial load
duke
parents:
diff changeset
   362
      set_idom(nnn, idom(nnn), dd-1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   363
    // While we're at it, remove any SafePoints from the peeled code
489c9b5090e2 Initial load
duke
parents:
diff changeset
   364
    if( old->Opcode() == Op_SafePoint ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   365
      Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   366
      lazy_replace(nnn,nnn->in(TypeFunc::Control));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   367
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   368
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   369
489c9b5090e2 Initial load
duke
parents:
diff changeset
   370
  // Now force out all loop-invariant dominating tests.  The optimizer
489c9b5090e2 Initial load
duke
parents:
diff changeset
   371
  // finds some, but we _know_ they are all useless.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   372
  peeled_dom_test_elim(loop,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   373
489c9b5090e2 Initial load
duke
parents:
diff changeset
   374
  loop->record_for_igvn();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   375
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   376
489c9b5090e2 Initial load
duke
parents:
diff changeset
   377
//------------------------------policy_maximally_unroll------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   378
// Return exact loop trip count, or 0 if not maximally unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   379
bool IdealLoopTree::policy_maximally_unroll( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   380
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   381
  assert( cl->is_normal_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   382
489c9b5090e2 Initial load
duke
parents:
diff changeset
   383
  Node *init_n = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   384
  Node *limit_n = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   385
489c9b5090e2 Initial load
duke
parents:
diff changeset
   386
  // Non-constant bounds
489c9b5090e2 Initial load
duke
parents:
diff changeset
   387
  if( init_n   == NULL || !init_n->is_Con()  ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   388
      limit_n  == NULL || !limit_n->is_Con() ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   389
      // protect against stride not being a constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   390
      !cl->stride_is_con() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   391
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   392
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   393
  int init   = init_n->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   394
  int limit  = limit_n->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   395
  int span   = limit - init;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   396
  int stride = cl->stride_con();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   397
489c9b5090e2 Initial load
duke
parents:
diff changeset
   398
  if (init >= limit || stride > span) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   399
    // return a false (no maximally unroll) and the regular unroll/peel
489c9b5090e2 Initial load
duke
parents:
diff changeset
   400
    // route will make a small mess which CCP will fold away.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   401
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   402
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   403
  uint trip_count = span/stride;   // trip_count can be greater than 2 Gig.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   404
  assert( (int)trip_count*stride == span, "must divide evenly" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   405
489c9b5090e2 Initial load
duke
parents:
diff changeset
   406
  // Real policy: if we maximally unroll, does it get too big?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   407
  // Allow the unrolled mess to get larger than standard loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   408
  // size.  After all, it will no longer be a loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   409
  uint body_size    = _body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   410
  uint unroll_limit = (uint)LoopUnrollLimit * 4;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   411
  assert( (intx)unroll_limit == LoopUnrollLimit * 4, "LoopUnrollLimit must fit in 32bits");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   412
  cl->set_trip_count(trip_count);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   413
  if( trip_count <= unroll_limit && body_size <= unroll_limit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   414
    uint new_body_size = body_size * trip_count;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   415
    if (new_body_size <= unroll_limit &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   416
        body_size == new_body_size / trip_count &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   417
        // Unrolling can result in a large amount of node construction
489c9b5090e2 Initial load
duke
parents:
diff changeset
   418
        new_body_size < MaxNodeLimit - phase->C->unique()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   419
      return true;    // maximally unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   420
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   421
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   422
489c9b5090e2 Initial load
duke
parents:
diff changeset
   423
  return false;               // Do not maximally unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   424
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   425
489c9b5090e2 Initial load
duke
parents:
diff changeset
   426
489c9b5090e2 Initial load
duke
parents:
diff changeset
   427
//------------------------------policy_unroll----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   428
// Return TRUE or FALSE if the loop should be unrolled or not.  Unroll if
489c9b5090e2 Initial load
duke
parents:
diff changeset
   429
// the loop is a CountedLoop and the body is small enough.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   430
bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   431
489c9b5090e2 Initial load
duke
parents:
diff changeset
   432
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   433
  assert( cl->is_normal_loop() || cl->is_main_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   434
489c9b5090e2 Initial load
duke
parents:
diff changeset
   435
  // protect against stride not being a constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   436
  if( !cl->stride_is_con() ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   437
489c9b5090e2 Initial load
duke
parents:
diff changeset
   438
  // protect against over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   439
  if( cl->trip_count() <= 1 ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   440
489c9b5090e2 Initial load
duke
parents:
diff changeset
   441
  int future_unroll_ct = cl->unrolled_count() * 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   442
489c9b5090e2 Initial load
duke
parents:
diff changeset
   443
  // Don't unroll if the next round of unrolling would push us
489c9b5090e2 Initial load
duke
parents:
diff changeset
   444
  // over the expected trip count of the loop.  One is subtracted
489c9b5090e2 Initial load
duke
parents:
diff changeset
   445
  // from the expected trip count because the pre-loop normally
489c9b5090e2 Initial load
duke
parents:
diff changeset
   446
  // executes 1 iteration.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   447
  if (UnrollLimitForProfileCheck > 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   448
      cl->profile_trip_cnt() != COUNT_UNKNOWN &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   449
      future_unroll_ct        > UnrollLimitForProfileCheck &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   450
      (float)future_unroll_ct > cl->profile_trip_cnt() - 1.0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   451
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   452
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   453
489c9b5090e2 Initial load
duke
parents:
diff changeset
   454
  // When unroll count is greater than LoopUnrollMin, don't unroll if:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   455
  //   the residual iterations are more than 10% of the trip count
489c9b5090e2 Initial load
duke
parents:
diff changeset
   456
  //   and rounds of "unroll,optimize" are not making significant progress
489c9b5090e2 Initial load
duke
parents:
diff changeset
   457
  //   Progress defined as current size less than 20% larger than previous size.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   458
  if (UseSuperWord && cl->node_count_before_unroll() > 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   459
      future_unroll_ct > LoopUnrollMin &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   460
      (future_unroll_ct - 1) * 10.0 > cl->profile_trip_cnt() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   461
      1.2 * cl->node_count_before_unroll() < (double)_body.size()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   462
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   463
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   464
489c9b5090e2 Initial load
duke
parents:
diff changeset
   465
  Node *init_n = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   466
  Node *limit_n = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   467
  // Non-constant bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   468
  // Protect against over-unrolling when init or/and limit are not constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   469
  // (so that trip_count's init value is maxint) but iv range is known.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   470
  if( init_n   == NULL || !init_n->is_Con()  ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   471
      limit_n  == NULL || !limit_n->is_Con() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   472
    Node* phi = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   473
    if( phi != NULL ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   474
      assert(phi->is_Phi() && phi->in(0) == _head, "Counted loop should have iv phi.");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   475
      const TypeInt* iv_type = phase->_igvn.type(phi)->is_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   476
      int next_stride = cl->stride_con() * 2; // stride after this unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   477
      if( next_stride > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   478
        if( iv_type->_lo + next_stride <= iv_type->_lo || // overflow
489c9b5090e2 Initial load
duke
parents:
diff changeset
   479
            iv_type->_lo + next_stride >  iv_type->_hi ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   480
          return false;  // over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   481
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   482
      } else if( next_stride < 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   483
        if( iv_type->_hi + next_stride >= iv_type->_hi || // overflow
489c9b5090e2 Initial load
duke
parents:
diff changeset
   484
            iv_type->_hi + next_stride <  iv_type->_lo ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   485
          return false;  // over-unrolling
489c9b5090e2 Initial load
duke
parents:
diff changeset
   486
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   487
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   488
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   489
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   490
489c9b5090e2 Initial load
duke
parents:
diff changeset
   491
  // Adjust body_size to determine if we unroll or not
489c9b5090e2 Initial load
duke
parents:
diff changeset
   492
  uint body_size = _body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   493
  // Key test to unroll CaffeineMark's Logic test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   494
  int xors_in_loop = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   495
  // Also count ModL, DivL and MulL which expand mightly
489c9b5090e2 Initial load
duke
parents:
diff changeset
   496
  for( uint k = 0; k < _body.size(); k++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   497
    switch( _body.at(k)->Opcode() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   498
    case Op_XorI: xors_in_loop++; break; // CaffeineMark's Logic test
489c9b5090e2 Initial load
duke
parents:
diff changeset
   499
    case Op_ModL: body_size += 30; break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   500
    case Op_DivL: body_size += 30; break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   501
    case Op_MulL: body_size += 10; break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   502
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   503
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   504
489c9b5090e2 Initial load
duke
parents:
diff changeset
   505
  // Check for being too big
489c9b5090e2 Initial load
duke
parents:
diff changeset
   506
  if( body_size > (uint)LoopUnrollLimit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   507
    if( xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   508
    // Normal case: loop too big
489c9b5090e2 Initial load
duke
parents:
diff changeset
   509
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   510
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   511
489c9b5090e2 Initial load
duke
parents:
diff changeset
   512
  // Check for stride being a small enough constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   513
  if( abs(cl->stride_con()) > (1<<3) ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   514
489c9b5090e2 Initial load
duke
parents:
diff changeset
   515
  // Unroll once!  (Each trip will soon do double iterations)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   516
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   517
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   518
489c9b5090e2 Initial load
duke
parents:
diff changeset
   519
//------------------------------policy_align-----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   520
// Return TRUE or FALSE if the loop should be cache-line aligned.  Gather the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   521
// expression that does the alignment.  Note that only one array base can be
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1500
diff changeset
   522
// aligned in a loop (unless the VM guarantees mutual alignment).  Note that
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   523
// if we vectorize short memory ops into longer memory ops, we may want to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   524
// increase alignment.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   525
bool IdealLoopTree::policy_align( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   526
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   527
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   528
489c9b5090e2 Initial load
duke
parents:
diff changeset
   529
//------------------------------policy_range_check-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   530
// Return TRUE or FALSE if the loop should be range-check-eliminated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   531
// Actually we do iteration-splitting, a more powerful form of RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   532
bool IdealLoopTree::policy_range_check( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   533
  if( !RangeCheckElimination ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   534
489c9b5090e2 Initial load
duke
parents:
diff changeset
   535
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   536
  // If we unrolled with no intention of doing RCE and we later
489c9b5090e2 Initial load
duke
parents:
diff changeset
   537
  // changed our minds, we got no pre-loop.  Either we need to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   538
  // make a new pre-loop, or we gotta disallow RCE.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   539
  if( cl->is_main_no_pre_loop() ) return false; // Disallowed for now.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   540
  Node *trip_counter = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   541
489c9b5090e2 Initial load
duke
parents:
diff changeset
   542
  // Check loop body for tests of trip-counter plus loop-invariant vs
489c9b5090e2 Initial load
duke
parents:
diff changeset
   543
  // loop-invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   544
  for( uint i = 0; i < _body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   545
    Node *iff = _body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   546
    if( iff->Opcode() == Op_If ) { // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   547
489c9b5090e2 Initial load
duke
parents:
diff changeset
   548
      // Comparing trip+off vs limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
   549
      Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   550
      if( bol->req() != 2 ) continue; // dead constant test
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   551
      if (!bol->is_Bool()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   552
        assert(UseLoopPredicate && bol->Opcode() == Op_Conv2B, "predicate check only");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   553
        continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   554
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   555
      Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   556
489c9b5090e2 Initial load
duke
parents:
diff changeset
   557
      Node *rc_exp = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   558
      Node *limit = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   559
489c9b5090e2 Initial load
duke
parents:
diff changeset
   560
      Node *limit_c = phase->get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   561
      if( limit_c == phase->C->top() )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   562
        return false;           // Found dead test on live IF?  No RCE!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   563
      if( is_member(phase->get_loop(limit_c) ) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   564
        // Compare might have operands swapped; commute them
489c9b5090e2 Initial load
duke
parents:
diff changeset
   565
        rc_exp = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   566
        limit  = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   567
        limit_c = phase->get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   568
        if( is_member(phase->get_loop(limit_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   569
          continue;             // Both inputs are loop varying; cannot RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
   570
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   571
489c9b5090e2 Initial load
duke
parents:
diff changeset
   572
      if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, NULL, NULL)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   573
        continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   574
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   575
      // Yeah!  Found a test like 'trip+off vs limit'
489c9b5090e2 Initial load
duke
parents:
diff changeset
   576
      // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   577
      // we need loop unswitching instead of iteration splitting.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   578
      if( is_loop_exit(iff) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   579
        return true;            // Found reason to split iterations
489c9b5090e2 Initial load
duke
parents:
diff changeset
   580
    } // End of is IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
   581
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   582
489c9b5090e2 Initial load
duke
parents:
diff changeset
   583
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   584
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   585
489c9b5090e2 Initial load
duke
parents:
diff changeset
   586
//------------------------------policy_peel_only-------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   587
// Return TRUE or FALSE if the loop should NEVER be RCE'd or aligned.  Useful
489c9b5090e2 Initial load
duke
parents:
diff changeset
   588
// for unrolling loops with NO array accesses.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   589
bool IdealLoopTree::policy_peel_only( PhaseIdealLoop *phase ) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   590
489c9b5090e2 Initial load
duke
parents:
diff changeset
   591
  for( uint i = 0; i < _body.size(); i++ )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   592
    if( _body[i]->is_Mem() )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   593
      return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   594
489c9b5090e2 Initial load
duke
parents:
diff changeset
   595
  // No memory accesses at all!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   596
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   597
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   598
489c9b5090e2 Initial load
duke
parents:
diff changeset
   599
//------------------------------clone_up_backedge_goo--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   600
// If Node n lives in the back_ctrl block and cannot float, we clone a private
489c9b5090e2 Initial load
duke
parents:
diff changeset
   601
// version of n in preheader_ctrl block and return that, otherwise return n.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   602
Node *PhaseIdealLoop::clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   603
  if( get_ctrl(n) != back_ctrl ) return n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   604
489c9b5090e2 Initial load
duke
parents:
diff changeset
   605
  Node *x = NULL;               // If required, a clone of 'n'
489c9b5090e2 Initial load
duke
parents:
diff changeset
   606
  // Check for 'n' being pinned in the backedge.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   607
  if( n->in(0) && n->in(0) == back_ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   608
    x = n->clone();             // Clone a copy of 'n' to preheader
489c9b5090e2 Initial load
duke
parents:
diff changeset
   609
    x->set_req( 0, preheader_ctrl ); // Fix x's control input to preheader
489c9b5090e2 Initial load
duke
parents:
diff changeset
   610
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   611
489c9b5090e2 Initial load
duke
parents:
diff changeset
   612
  // Recursive fixup any other input edges into x.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   613
  // If there are no changes we can just return 'n', otherwise
489c9b5090e2 Initial load
duke
parents:
diff changeset
   614
  // we need to clone a private copy and change it.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   615
  for( uint i = 1; i < n->req(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   616
    Node *g = clone_up_backedge_goo( back_ctrl, preheader_ctrl, n->in(i) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   617
    if( g != n->in(i) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   618
      if( !x )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   619
        x = n->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   620
      x->set_req(i, g);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   621
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   622
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   623
  if( x ) {                     // x can legally float to pre-header location
489c9b5090e2 Initial load
duke
parents:
diff changeset
   624
    register_new_node( x, preheader_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   625
    return x;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   626
  } else {                      // raise n to cover LCA of uses
489c9b5090e2 Initial load
duke
parents:
diff changeset
   627
    set_ctrl( n, find_non_split_ctrl(back_ctrl->in(0)) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   628
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   629
  return n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   630
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   631
489c9b5090e2 Initial load
duke
parents:
diff changeset
   632
//------------------------------insert_pre_post_loops--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   633
// Insert pre and post loops.  If peel_only is set, the pre-loop can not have
489c9b5090e2 Initial load
duke
parents:
diff changeset
   634
// more iterations added.  It acts as a 'peel' only, no lower-bound RCE, no
489c9b5090e2 Initial load
duke
parents:
diff changeset
   635
// alignment.  Useful to unroll loops that do no array accesses.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   636
void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   637
489c9b5090e2 Initial load
duke
parents:
diff changeset
   638
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   639
489c9b5090e2 Initial load
duke
parents:
diff changeset
   640
  // Find common pieces of the loop being guarded with pre & post loops
489c9b5090e2 Initial load
duke
parents:
diff changeset
   641
  CountedLoopNode *main_head = loop->_head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   642
  assert( main_head->is_normal_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   643
  CountedLoopEndNode *main_end = main_head->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   644
  assert( main_end->outcnt() == 2, "1 true, 1 false path only" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   645
  uint dd_main_head = dom_depth(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   646
  uint max = main_head->outcnt();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   647
489c9b5090e2 Initial load
duke
parents:
diff changeset
   648
  Node *pre_header= main_head->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   649
  Node *init      = main_head->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   650
  Node *incr      = main_end ->incr();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   651
  Node *limit     = main_end ->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   652
  Node *stride    = main_end ->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   653
  Node *cmp       = main_end ->cmp_node();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   654
  BoolTest::mask b_test = main_end->test_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   655
489c9b5090e2 Initial load
duke
parents:
diff changeset
   656
  // Need only 1 user of 'bol' because I will be hacking the loop bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   657
  Node *bol = main_end->in(CountedLoopEndNode::TestValue);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   658
  if( bol->outcnt() != 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   659
    bol = bol->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   660
    register_new_node(bol,main_end->in(CountedLoopEndNode::TestControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   661
    _igvn.hash_delete(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   662
    main_end->set_req(CountedLoopEndNode::TestValue, bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   663
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   664
  // Need only 1 user of 'cmp' because I will be hacking the loop bounds.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   665
  if( cmp->outcnt() != 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   666
    cmp = cmp->clone();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   667
    register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   668
    _igvn.hash_delete(bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   669
    bol->set_req(1, cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   670
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   671
489c9b5090e2 Initial load
duke
parents:
diff changeset
   672
  //------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   673
  // Step A: Create Post-Loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   674
  Node* main_exit = main_end->proj_out(false);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   675
  assert( main_exit->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   676
  int dd_main_exit = dom_depth(main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   677
489c9b5090e2 Initial load
duke
parents:
diff changeset
   678
  // Step A1: Clone the loop body.  The clone becomes the post-loop.  The main
489c9b5090e2 Initial load
duke
parents:
diff changeset
   679
  // loop pre-header illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   680
  clone_loop( loop, old_new, dd_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   681
  assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   682
  CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   683
  post_head->set_post_loop(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   684
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   685
  // Reduce the post-loop trip count.
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   686
  CountedLoopEndNode* post_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   687
  post_end->_prob = PROB_FAIR;
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   688
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   689
  // Build the main-loop normal exit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   690
  IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   691
  _igvn.register_new_node_with_optimizer( new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   692
  set_idom(new_main_exit, main_end, dd_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   693
  set_loop(new_main_exit, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   694
489c9b5090e2 Initial load
duke
parents:
diff changeset
   695
  // Step A2: Build a zero-trip guard for the post-loop.  After leaving the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   696
  // main-loop, the post-loop may not execute at all.  We 'opaque' the incr
489c9b5090e2 Initial load
duke
parents:
diff changeset
   697
  // (the main-loop trip-counter exit value) because we will be changing
489c9b5090e2 Initial load
duke
parents:
diff changeset
   698
  // the exit value (via unrolling) so we cannot constant-fold away the zero
489c9b5090e2 Initial load
duke
parents:
diff changeset
   699
  // trip guard until all unrolling is done.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   700
  Node *zer_opaq = new (C, 2) Opaque1Node(C, incr);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   701
  Node *zer_cmp  = new (C, 3) CmpINode( zer_opaq, limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   702
  Node *zer_bol  = new (C, 2) BoolNode( zer_cmp, b_test );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   703
  register_new_node( zer_opaq, new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   704
  register_new_node( zer_cmp , new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   705
  register_new_node( zer_bol , new_main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   706
489c9b5090e2 Initial load
duke
parents:
diff changeset
   707
  // Build the IfNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
   708
  IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   709
  _igvn.register_new_node_with_optimizer( zer_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   710
  set_idom(zer_iff, new_main_exit, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   711
  set_loop(zer_iff, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   712
489c9b5090e2 Initial load
duke
parents:
diff changeset
   713
  // Plug in the false-path, taken if we need to skip post-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   714
  _igvn.hash_delete( main_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   715
  main_exit->set_req(0, zer_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   716
  _igvn._worklist.push(main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   717
  set_idom(main_exit, zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   718
  set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   719
  // Make the true-path, must enter the post loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   720
  Node *zer_taken = new (C, 1) IfTrueNode( zer_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   721
  _igvn.register_new_node_with_optimizer( zer_taken );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   722
  set_idom(zer_taken, zer_iff, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   723
  set_loop(zer_taken, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   724
  // Plug in the true path
489c9b5090e2 Initial load
duke
parents:
diff changeset
   725
  _igvn.hash_delete( post_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   726
  post_head->set_req(LoopNode::EntryControl, zer_taken);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   727
  set_idom(post_head, zer_taken, dd_main_exit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   728
489c9b5090e2 Initial load
duke
parents:
diff changeset
   729
  // Step A3: Make the fall-in values to the post-loop come from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   730
  // fall-out values of the main-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   731
  for (DUIterator_Fast imax, i = main_head->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   732
    Node* main_phi = main_head->fast_out(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   733
    if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() >0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   734
      Node *post_phi = old_new[main_phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   735
      Node *fallmain  = clone_up_backedge_goo(main_head->back_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   736
                                              post_head->init_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   737
                                              main_phi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   738
      _igvn.hash_delete(post_phi);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   739
      post_phi->set_req( LoopNode::EntryControl, fallmain );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   740
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   741
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   742
489c9b5090e2 Initial load
duke
parents:
diff changeset
   743
  // Update local caches for next stanza
489c9b5090e2 Initial load
duke
parents:
diff changeset
   744
  main_exit = new_main_exit;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   745
489c9b5090e2 Initial load
duke
parents:
diff changeset
   746
489c9b5090e2 Initial load
duke
parents:
diff changeset
   747
  //------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   748
  // Step B: Create Pre-Loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   749
489c9b5090e2 Initial load
duke
parents:
diff changeset
   750
  // Step B1: Clone the loop body.  The clone becomes the pre-loop.  The main
489c9b5090e2 Initial load
duke
parents:
diff changeset
   751
  // loop pre-header illegally has 2 control users (old & new loops).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   752
  clone_loop( loop, old_new, dd_main_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   753
  CountedLoopNode*    pre_head = old_new[main_head->_idx]->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   754
  CountedLoopEndNode* pre_end  = old_new[main_end ->_idx]->as_CountedLoopEnd();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   755
  pre_head->set_pre_loop(main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   756
  Node *pre_incr = old_new[incr->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   757
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   758
  // Reduce the pre-loop trip count.
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   759
  pre_end->_prob = PROB_FAIR;
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   760
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   761
  // Find the pre-loop normal exit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   762
  Node* pre_exit = pre_end->proj_out(false);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   763
  assert( pre_exit->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   764
  IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   765
  _igvn.register_new_node_with_optimizer( new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   766
  set_idom(new_pre_exit, pre_end, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   767
  set_loop(new_pre_exit, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   768
489c9b5090e2 Initial load
duke
parents:
diff changeset
   769
  // Step B2: Build a zero-trip guard for the main-loop.  After leaving the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   770
  // pre-loop, the main-loop may not execute at all.  Later in life this
489c9b5090e2 Initial load
duke
parents:
diff changeset
   771
  // zero-trip guard will become the minimum-trip guard when we unroll
489c9b5090e2 Initial load
duke
parents:
diff changeset
   772
  // the main-loop.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   773
  Node *min_opaq = new (C, 2) Opaque1Node(C, limit);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   774
  Node *min_cmp  = new (C, 3) CmpINode( pre_incr, min_opaq );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   775
  Node *min_bol  = new (C, 2) BoolNode( min_cmp, b_test );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   776
  register_new_node( min_opaq, new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   777
  register_new_node( min_cmp , new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   778
  register_new_node( min_bol , new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   779
1432
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   780
  // Build the IfNode (assume the main-loop is executed always).
44f076e3d2a4 6667595: Set probability FAIR for pre-, post- loops and ALWAYS for main loop
kvn
parents: 1399
diff changeset
   781
  IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_ALWAYS, COUNT_UNKNOWN );
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   782
  _igvn.register_new_node_with_optimizer( min_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   783
  set_idom(min_iff, new_pre_exit, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   784
  set_loop(min_iff, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   785
489c9b5090e2 Initial load
duke
parents:
diff changeset
   786
  // Plug in the false-path, taken if we need to skip main-loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   787
  _igvn.hash_delete( pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   788
  pre_exit->set_req(0, min_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   789
  set_idom(pre_exit, min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   790
  set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   791
  // Make the true-path, must enter the main loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   792
  Node *min_taken = new (C, 1) IfTrueNode( min_iff );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   793
  _igvn.register_new_node_with_optimizer( min_taken );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   794
  set_idom(min_taken, min_iff, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   795
  set_loop(min_taken, loop->_parent);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   796
  // Plug in the true path
489c9b5090e2 Initial load
duke
parents:
diff changeset
   797
  _igvn.hash_delete( main_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   798
  main_head->set_req(LoopNode::EntryControl, min_taken);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   799
  set_idom(main_head, min_taken, dd_main_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   800
489c9b5090e2 Initial load
duke
parents:
diff changeset
   801
  // Step B3: Make the fall-in values to the main-loop come from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   802
  // fall-out values of the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   803
  for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   804
    Node* main_phi = main_head->fast_out(i2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   805
    if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   806
      Node *pre_phi = old_new[main_phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   807
      Node *fallpre  = clone_up_backedge_goo(pre_head->back_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   808
                                             main_head->init_control(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   809
                                             pre_phi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   810
      _igvn.hash_delete(main_phi);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   811
      main_phi->set_req( LoopNode::EntryControl, fallpre );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   812
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   813
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   814
489c9b5090e2 Initial load
duke
parents:
diff changeset
   815
  // Step B4: Shorten the pre-loop to run only 1 iteration (for now).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   816
  // RCE and alignment may change this later.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   817
  Node *cmp_end = pre_end->cmp_node();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   818
  assert( cmp_end->in(2) == limit, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   819
  Node *pre_limit = new (C, 3) AddINode( init, stride );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   820
489c9b5090e2 Initial load
duke
parents:
diff changeset
   821
  // Save the original loop limit in this Opaque1 node for
489c9b5090e2 Initial load
duke
parents:
diff changeset
   822
  // use by range check elimination.
762
1b26adb5fea1 6715633: when matching a memory node the adr_type should not change
kvn
parents: 360
diff changeset
   823
  Node *pre_opaq  = new (C, 3) Opaque1Node(C, pre_limit, limit);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   824
489c9b5090e2 Initial load
duke
parents:
diff changeset
   825
  register_new_node( pre_limit, pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   826
  register_new_node( pre_opaq , pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   827
489c9b5090e2 Initial load
duke
parents:
diff changeset
   828
  // Since no other users of pre-loop compare, I can hack limit directly
489c9b5090e2 Initial load
duke
parents:
diff changeset
   829
  assert( cmp_end->outcnt() == 1, "no other users" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   830
  _igvn.hash_delete(cmp_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   831
  cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   832
489c9b5090e2 Initial load
duke
parents:
diff changeset
   833
  // Special case for not-equal loop bounds:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   834
  // Change pre loop test, main loop test, and the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   835
  // main loop guard test to use lt or gt depending on stride
489c9b5090e2 Initial load
duke
parents:
diff changeset
   836
  // direction:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   837
  // positive stride use <
489c9b5090e2 Initial load
duke
parents:
diff changeset
   838
  // negative stride use >
489c9b5090e2 Initial load
duke
parents:
diff changeset
   839
489c9b5090e2 Initial load
duke
parents:
diff changeset
   840
  if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   841
489c9b5090e2 Initial load
duke
parents:
diff changeset
   842
    BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   843
    // Modify pre loop end condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
   844
    Node* pre_bol = pre_end->in(CountedLoopEndNode::TestValue)->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   845
    BoolNode* new_bol0 = new (C, 2) BoolNode(pre_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   846
    register_new_node( new_bol0, pre_head->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   847
    _igvn.hash_delete(pre_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   848
    pre_end->set_req(CountedLoopEndNode::TestValue, new_bol0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   849
    // Modify main loop guard condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
   850
    assert(min_iff->in(CountedLoopEndNode::TestValue) == min_bol, "guard okay");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   851
    BoolNode* new_bol1 = new (C, 2) BoolNode(min_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   852
    register_new_node( new_bol1, new_pre_exit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   853
    _igvn.hash_delete(min_iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   854
    min_iff->set_req(CountedLoopEndNode::TestValue, new_bol1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   855
    // Modify main loop end condition
489c9b5090e2 Initial load
duke
parents:
diff changeset
   856
    BoolNode* main_bol = main_end->in(CountedLoopEndNode::TestValue)->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   857
    BoolNode* new_bol2 = new (C, 2) BoolNode(main_bol->in(1), new_test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   858
    register_new_node( new_bol2, main_end->in(CountedLoopEndNode::TestControl) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   859
    _igvn.hash_delete(main_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   860
    main_end->set_req(CountedLoopEndNode::TestValue, new_bol2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   861
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   862
489c9b5090e2 Initial load
duke
parents:
diff changeset
   863
  // Flag main loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
   864
  main_head->set_main_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   865
  if( peel_only ) main_head->set_main_no_pre_loop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   866
489c9b5090e2 Initial load
duke
parents:
diff changeset
   867
  // It's difficult to be precise about the trip-counts
489c9b5090e2 Initial load
duke
parents:
diff changeset
   868
  // for the pre/post loops.  They are usually very short,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   869
  // so guess that 4 trips is a reasonable value.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   870
  post_head->set_profile_trip_cnt(4.0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   871
  pre_head->set_profile_trip_cnt(4.0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   872
489c9b5090e2 Initial load
duke
parents:
diff changeset
   873
  // Now force out all loop-invariant dominating tests.  The optimizer
489c9b5090e2 Initial load
duke
parents:
diff changeset
   874
  // finds some, but we _know_ they are all useless.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   875
  peeled_dom_test_elim(loop,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   876
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   877
489c9b5090e2 Initial load
duke
parents:
diff changeset
   878
//------------------------------is_invariant-----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   879
// Return true if n is invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
   880
bool IdealLoopTree::is_invariant(Node* n) const {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
   881
  Node *n_c = _phase->has_ctrl(n) ? _phase->get_ctrl(n) : n;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   882
  if (n_c->is_top()) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   883
  return !is_member(_phase->get_loop(n_c));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   884
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   885
489c9b5090e2 Initial load
duke
parents:
diff changeset
   886
489c9b5090e2 Initial load
duke
parents:
diff changeset
   887
//------------------------------do_unroll--------------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   888
// Unroll the loop body one step - make each trip do 2 iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   889
void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   890
  assert( LoopUnrollLimit, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   891
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   892
  if( PrintOpto && VerifyLoopOptimizations ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   893
    tty->print("Unrolling ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   894
    loop->dump_head();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   895
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   896
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
   897
  CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   898
  CountedLoopEndNode *loop_end = loop_head->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   899
  assert( loop_end, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   900
489c9b5090e2 Initial load
duke
parents:
diff changeset
   901
  // Remember loop node count before unrolling to detect
489c9b5090e2 Initial load
duke
parents:
diff changeset
   902
  // if rounds of unroll,optimize are making progress
489c9b5090e2 Initial load
duke
parents:
diff changeset
   903
  loop_head->set_node_count_before_unroll(loop->_body.size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   904
489c9b5090e2 Initial load
duke
parents:
diff changeset
   905
  Node *ctrl  = loop_head->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   906
  Node *limit = loop_head->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   907
  Node *init  = loop_head->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   908
  Node *strid = loop_head->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   909
489c9b5090e2 Initial load
duke
parents:
diff changeset
   910
  Node *opaq = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   911
  if( adjust_min_trip ) {       // If not maximally unrolling, need adjustment
489c9b5090e2 Initial load
duke
parents:
diff changeset
   912
    assert( loop_head->is_main_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   913
    assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   914
    Node *iff = ctrl->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   915
    assert( iff->Opcode() == Op_If, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   916
    Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   917
    assert( bol->Opcode() == Op_Bool, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   918
    Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   919
    assert( cmp->Opcode() == Op_CmpI, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   920
    opaq = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   921
    // Occasionally it's possible for a pre-loop Opaque1 node to be
489c9b5090e2 Initial load
duke
parents:
diff changeset
   922
    // optimized away and then another round of loop opts attempted.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   923
    // We can not optimize this particular loop in that case.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   924
    if( opaq->Opcode() != Op_Opaque1 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
   925
      return;                   // Cannot find pre-loop!  Bail out!
489c9b5090e2 Initial load
duke
parents:
diff changeset
   926
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   927
489c9b5090e2 Initial load
duke
parents:
diff changeset
   928
  C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   929
489c9b5090e2 Initial load
duke
parents:
diff changeset
   930
  // Adjust max trip count. The trip count is intentionally rounded
489c9b5090e2 Initial load
duke
parents:
diff changeset
   931
  // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   932
  // the main, unrolled, part of the loop will never execute as it is protected
489c9b5090e2 Initial load
duke
parents:
diff changeset
   933
  // by the min-trip test.  See bug 4834191 for a case where we over-unrolled
489c9b5090e2 Initial load
duke
parents:
diff changeset
   934
  // and later determined that part of the unrolled loop was dead.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   935
  loop_head->set_trip_count(loop_head->trip_count() / 2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   936
489c9b5090e2 Initial load
duke
parents:
diff changeset
   937
  // Double the count of original iterations in the unrolled loop body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   938
  loop_head->double_unrolled_count();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   939
489c9b5090e2 Initial load
duke
parents:
diff changeset
   940
  // -----------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   941
  // Step 2: Cut back the trip counter for an unroll amount of 2.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   942
  // Loop will normally trip (limit - init)/stride_con.  Since it's a
489c9b5090e2 Initial load
duke
parents:
diff changeset
   943
  // CountedLoop this is exact (stride divides limit-init exactly).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   944
  // We are going to double the loop body, so we want to knock off any
489c9b5090e2 Initial load
duke
parents:
diff changeset
   945
  // odd iteration: (trip_cnt & ~1).  Then back compute a new limit.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   946
  Node *span = new (C, 3) SubINode( limit, init );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   947
  register_new_node( span, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   948
  Node *trip = new (C, 3) DivINode( 0, span, strid );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   949
  register_new_node( trip, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   950
  Node *mtwo = _igvn.intcon(-2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   951
  set_ctrl(mtwo, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   952
  Node *rond = new (C, 3) AndINode( trip, mtwo );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   953
  register_new_node( rond, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   954
  Node *spn2 = new (C, 3) MulINode( rond, strid );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   955
  register_new_node( spn2, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   956
  Node *lim2 = new (C, 3) AddINode( spn2, init );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   957
  register_new_node( lim2, ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   958
489c9b5090e2 Initial load
duke
parents:
diff changeset
   959
  // Hammer in the new limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
   960
  Node *ctrl2 = loop_end->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   961
  Node *cmp2 = new (C, 3) CmpINode( loop_head->incr(), lim2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   962
  register_new_node( cmp2, ctrl2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   963
  Node *bol2 = new (C, 2) BoolNode( cmp2, loop_end->test_trip() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   964
  register_new_node( bol2, ctrl2 );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   965
  _igvn.hash_delete(loop_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   966
  loop_end->set_req(CountedLoopEndNode::TestValue, bol2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   967
489c9b5090e2 Initial load
duke
parents:
diff changeset
   968
  // Step 3: Find the min-trip test guaranteed before a 'main' loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   969
  // Make it a 1-trip test (means at least 2 trips).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   970
  if( adjust_min_trip ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   971
    // Guard test uses an 'opaque' node which is not shared.  Hence I
489c9b5090e2 Initial load
duke
parents:
diff changeset
   972
    // can edit it's inputs directly.  Hammer in the new limit for the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   973
    // minimum-trip guard.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   974
    assert( opaq->outcnt() == 1, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   975
    _igvn.hash_delete(opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   976
    opaq->set_req(1, lim2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   977
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   978
489c9b5090e2 Initial load
duke
parents:
diff changeset
   979
  // ---------
489c9b5090e2 Initial load
duke
parents:
diff changeset
   980
  // Step 4: Clone the loop body.  Move it inside the loop.  This loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
   981
  // represents the odd iterations; since the loop trips an even number of
489c9b5090e2 Initial load
duke
parents:
diff changeset
   982
  // times its backedge is never taken.  Kill the backedge.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   983
  uint dd = dom_depth(loop_head);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   984
  clone_loop( loop, old_new, dd );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   985
489c9b5090e2 Initial load
duke
parents:
diff changeset
   986
  // Make backedges of the clone equal to backedges of the original.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   987
  // Make the fall-in from the original come from the fall-out of the clone.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   988
  for (DUIterator_Fast jmax, j = loop_head->fast_outs(jmax); j < jmax; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   989
    Node* phi = loop_head->fast_out(j);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   990
    if( phi->is_Phi() && phi->in(0) == loop_head && phi->outcnt() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   991
      Node *newphi = old_new[phi->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   992
      _igvn.hash_delete( phi );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   993
      _igvn.hash_delete( newphi );
489c9b5090e2 Initial load
duke
parents:
diff changeset
   994
489c9b5090e2 Initial load
duke
parents:
diff changeset
   995
      phi   ->set_req(LoopNode::   EntryControl, newphi->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   996
      newphi->set_req(LoopNode::LoopBackControl, phi   ->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   997
      phi   ->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   998
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   999
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1000
  Node *clone_head = old_new[loop_head->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1001
  _igvn.hash_delete( clone_head );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1002
  loop_head ->set_req(LoopNode::   EntryControl, clone_head->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1003
  clone_head->set_req(LoopNode::LoopBackControl, loop_head ->in(LoopNode::LoopBackControl));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1004
  loop_head ->set_req(LoopNode::LoopBackControl, C->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1005
  loop->_head = clone_head;     // New loop header
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1006
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1007
  set_idom(loop_head,  loop_head ->in(LoopNode::EntryControl), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1008
  set_idom(clone_head, clone_head->in(LoopNode::EntryControl), dd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1009
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1010
  // Kill the clone's backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1011
  Node *newcle = old_new[loop_end->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1012
  _igvn.hash_delete( newcle );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1013
  Node *one = _igvn.intcon(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1014
  set_ctrl(one, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1015
  newcle->set_req(1, one);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1016
  // Force clone into same loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1017
  uint max = loop->_body.size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1018
  for( uint k = 0; k < max; k++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1019
    Node *old = loop->_body.at(k);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1020
    Node *nnn = old_new[old->_idx];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1021
    loop->_body.push(nnn);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1022
    if (!has_ctrl(old))
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1023
      set_loop(nnn, loop);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1024
  }
1399
9648dfd4ce09 6384206: Phis which are later unneeded are impairing our ability to inline based on static types
never
parents: 781
diff changeset
  1025
9648dfd4ce09 6384206: Phis which are later unneeded are impairing our ability to inline based on static types
never
parents: 781
diff changeset
  1026
  loop->record_for_igvn();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1027
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1028
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1029
//------------------------------do_maximally_unroll----------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1030
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1031
void PhaseIdealLoop::do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1032
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1033
  assert( cl->trip_count() > 0, "");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1034
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1035
  // If loop is tripping an odd number of times, peel odd iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1036
  if( (cl->trip_count() & 1) == 1 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1037
    do_peeling( loop, old_new );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1038
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1039
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1040
  // Now its tripping an even number of times remaining.  Double loop body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1041
  // Do not adjust pre-guards; they are not needed and do not exist.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1042
  if( cl->trip_count() > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1043
    do_unroll( loop, old_new, false );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1044
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1045
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1046
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1047
//------------------------------dominates_backedge---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1048
// Returns true if ctrl is executed on every complete iteration
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1049
bool IdealLoopTree::dominates_backedge(Node* ctrl) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1050
  assert(ctrl->is_CFG(), "must be control");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1051
  Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1052
  return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1053
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1054
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1055
//------------------------------add_constraint---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1056
// Constrain the main loop iterations so the condition:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1057
//    scale_con * I + offset  <  limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1058
// always holds true.  That is, either increase the number of iterations in
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1059
// the pre-loop or the post-loop until the condition holds true in the main
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1060
// loop.  Stride, scale, offset and limit are all loop invariant.  Further,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1061
// stride and scale are constants (offset and limit often are).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1062
void PhaseIdealLoop::add_constraint( int stride_con, int scale_con, Node *offset, Node *limit, Node *pre_ctrl, Node **pre_limit, Node **main_limit ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1063
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1064
  // Compute "I :: (limit-offset)/scale_con"
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1065
  Node *con = new (C, 3) SubINode( limit, offset );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1066
  register_new_node( con, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1067
  Node *scale = _igvn.intcon(scale_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1068
  set_ctrl(scale, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1069
  Node *X = new (C, 3) DivINode( 0, con, scale );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1070
  register_new_node( X, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1071
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1072
  // For positive stride, the pre-loop limit always uses a MAX function
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1073
  // and the main loop a MIN function.  For negative stride these are
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1074
  // reversed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1075
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1076
  // Also for positive stride*scale the affine function is increasing, so the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1077
  // pre-loop must check for underflow and the post-loop for overflow.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1078
  // Negative stride*scale reverses this; pre-loop checks for overflow and
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1079
  // post-loop for underflow.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1080
  if( stride_con*scale_con > 0 ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1081
    // Compute I < (limit-offset)/scale_con
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1082
    // Adjust main-loop last iteration to be MIN/MAX(main_loop,X)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1083
    *main_limit = (stride_con > 0)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1084
      ? (Node*)(new (C, 3) MinINode( *main_limit, X ))
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1085
      : (Node*)(new (C, 3) MaxINode( *main_limit, X ));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1086
    register_new_node( *main_limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1087
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1088
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1089
    // Compute (limit-offset)/scale_con + SGN(-scale_con) <= I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1090
    // Add the negation of the main-loop constraint to the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1091
    // See footnote [++] below for a derivation of the limit expression.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1092
    Node *incr = _igvn.intcon(scale_con > 0 ? -1 : 1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1093
    set_ctrl(incr, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1094
    Node *adj = new (C, 3) AddINode( X, incr );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1095
    register_new_node( adj, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1096
    *pre_limit = (scale_con > 0)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1097
      ? (Node*)new (C, 3) MinINode( *pre_limit, adj )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1098
      : (Node*)new (C, 3) MaxINode( *pre_limit, adj );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1099
    register_new_node( *pre_limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1100
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1101
//   [++] Here's the algebra that justifies the pre-loop limit expression:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1102
//
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1103
//   NOT( scale_con * I + offset  <  limit )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1104
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1105
//   scale_con * I + offset  >=  limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1106
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1107
//   SGN(scale_con) * I  >=  (limit-offset)/|scale_con|
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1108
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1109
//   (limit-offset)/|scale_con|   <=  I * SGN(scale_con)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1110
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1111
//   (limit-offset)/|scale_con|-1  <  I * SGN(scale_con)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1112
//      ==
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1113
//   ( if (scale_con > 0) /*common case*/
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1114
//       (limit-offset)/scale_con - 1  <  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1115
//     else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1116
//       (limit-offset)/scale_con + 1  >  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1117
//    )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1118
//   ( if (scale_con > 0) /*common case*/
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1119
//       (limit-offset)/scale_con + SGN(-scale_con)  <  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1120
//     else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1121
//       (limit-offset)/scale_con + SGN(-scale_con)  >  I
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1122
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1123
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1124
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1125
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1126
//------------------------------is_scaled_iv---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1127
// Return true if exp is a constant times an induction var
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1128
bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1129
  if (exp == iv) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1130
    if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1131
      *p_scale = 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1132
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1133
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1134
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1135
  int opc = exp->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1136
  if (opc == Op_MulI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1137
    if (exp->in(1) == iv && exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1138
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1139
        *p_scale = exp->in(2)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1140
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1141
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1142
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1143
    if (exp->in(2) == iv && exp->in(1)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1144
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1145
        *p_scale = exp->in(1)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1146
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1147
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1148
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1149
  } else if (opc == Op_LShiftI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1150
    if (exp->in(1) == iv && exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1151
      if (p_scale != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1152
        *p_scale = 1 << exp->in(2)->get_int();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1153
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1154
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1155
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1156
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1157
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1158
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1159
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1160
//-----------------------------is_scaled_iv_plus_offset------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1161
// Return true if exp is a simple induction variable expression: k1*iv + (invar + k2)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1162
bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1163
  if (is_scaled_iv(exp, iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1164
    if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1165
      Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1166
      set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1167
      *p_offset = zero;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1168
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1169
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1170
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1171
  int opc = exp->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1172
  if (opc == Op_AddI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1173
    if (is_scaled_iv(exp->in(1), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1174
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1175
        *p_offset = exp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1176
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1177
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1178
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1179
    if (exp->in(2)->is_Con()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1180
      Node* offset2 = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1181
      if (depth < 2 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1182
          is_scaled_iv_plus_offset(exp->in(1), iv, p_scale,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1183
                                   p_offset != NULL ? &offset2 : NULL, depth+1)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1184
        if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1185
          Node *ctrl_off2 = get_ctrl(offset2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1186
          Node* offset = new (C, 3) AddINode(offset2, exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1187
          register_new_node(offset, ctrl_off2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1188
          *p_offset = offset;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1189
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1190
        return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1191
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1192
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1193
  } else if (opc == Op_SubI) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1194
    if (is_scaled_iv(exp->in(1), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1195
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1196
        Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1197
        set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1198
        Node *ctrl_off = get_ctrl(exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1199
        Node* offset = new (C, 3) SubINode(zero, exp->in(2));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1200
        register_new_node(offset, ctrl_off);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1201
        *p_offset = offset;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1202
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1203
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1204
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1205
    if (is_scaled_iv(exp->in(2), iv, p_scale)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1206
      if (p_offset != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1207
        *p_scale *= -1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1208
        *p_offset = exp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1209
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1210
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1211
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1212
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1213
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1214
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1215
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1216
//------------------------------do_range_check---------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1217
// Eliminate range-checks and other trip-counter vs loop-invariant tests.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1218
void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1219
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1220
  if( PrintOpto && VerifyLoopOptimizations ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1221
    tty->print("Range Check Elimination ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1222
    loop->dump_head();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1223
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1224
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1225
  assert( RangeCheckElimination, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1226
  CountedLoopNode *cl = loop->_head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1227
  assert( cl->is_main_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1228
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1229
  // Find the trip counter; we are iteration splitting based on it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1230
  Node *trip_counter = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1231
  // Find the main loop limit; we will trim it's iterations
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1232
  // to not ever trip end tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1233
  Node *main_limit = cl->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1234
  // Find the pre-loop limit; we will expand it's iterations to
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1235
  // not ever trip low tests.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1236
  Node *ctrl  = cl->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1237
  assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1238
  Node *iffm = ctrl->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1239
  assert( iffm->Opcode() == Op_If, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1240
  Node *p_f = iffm->in(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1241
  assert( p_f->Opcode() == Op_IfFalse, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1242
  CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1243
  assert( pre_end->loopnode()->is_pre_loop(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1244
  Node *pre_opaq1 = pre_end->limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1245
  // Occasionally it's possible for a pre-loop Opaque1 node to be
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1246
  // optimized away and then another round of loop opts attempted.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1247
  // We can not optimize this particular loop in that case.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1248
  if( pre_opaq1->Opcode() != Op_Opaque1 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1249
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1250
  Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1251
  Node *pre_limit = pre_opaq->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1252
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1253
  // Where do we put new limit calculations
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1254
  Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1255
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1256
  // Ensure the original loop limit is available from the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1257
  // pre-loop Opaque1 node.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1258
  Node *orig_limit = pre_opaq->original_loop_limit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1259
  if( orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1260
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1261
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1262
  // Need to find the main-loop zero-trip guard
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1263
  Node *bolzm = iffm->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1264
  assert( bolzm->Opcode() == Op_Bool, "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1265
  Node *cmpzm = bolzm->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1266
  assert( cmpzm->is_Cmp(), "" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1267
  Node *opqzm = cmpzm->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1268
  if( opqzm->Opcode() != Op_Opaque1 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1269
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1270
  assert( opqzm->in(1) == main_limit, "do not understand situation" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1271
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1272
  // Must know if its a count-up or count-down loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1273
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1274
  // protect against stride not being a constant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1275
  if ( !cl->stride_is_con() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1276
    return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1277
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1278
  int stride_con = cl->stride_con();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1279
  Node *zero = _igvn.intcon(0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1280
  Node *one  = _igvn.intcon(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1281
  set_ctrl(zero, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1282
  set_ctrl(one,  C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1283
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1284
  // Range checks that do not dominate the loop backedge (ie.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1285
  // conditionally executed) can lengthen the pre loop limit beyond
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1286
  // the original loop limit. To prevent this, the pre limit is
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1287
  // (for stride > 0) MINed with the original loop limit (MAXed
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1288
  // stride < 0) when some range_check (rc) is conditionally
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1289
  // executed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1290
  bool conditional_rc = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1291
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1292
  // Check loop body for tests of trip-counter plus loop-invariant vs
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1293
  // loop-invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1294
  for( uint i = 0; i < loop->_body.size(); i++ ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1295
    Node *iff = loop->_body[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1296
    if( iff->Opcode() == Op_If ) { // Test?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1297
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1298
      // Test is an IfNode, has 2 projections.  If BOTH are in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1299
      // we need loop unswitching instead of iteration splitting.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1300
      Node *exit = loop->is_loop_exit(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1301
      if( !exit ) continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1302
      int flip = (exit->Opcode() == Op_IfTrue) ? 1 : 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1303
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1304
      // Get boolean condition to test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1305
      Node *i1 = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1306
      if( !i1->is_Bool() ) continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1307
      BoolNode *bol = i1->as_Bool();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1308
      BoolTest b_test = bol->_test;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1309
      // Flip sense of test if exit condition is flipped
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1310
      if( flip )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1311
        b_test = b_test.negate();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1312
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1313
      // Get compare
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1314
      Node *cmp = bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1315
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1316
      // Look for trip_counter + offset vs limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1317
      Node *rc_exp = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1318
      Node *limit  = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1319
      jint scale_con= 1;        // Assume trip counter not scaled
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1320
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1321
      Node *limit_c = get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1322
      if( loop->is_member(get_loop(limit_c) ) ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1323
        // Compare might have operands swapped; commute them
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1324
        b_test = b_test.commute();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1325
        rc_exp = cmp->in(2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1326
        limit  = cmp->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1327
        limit_c = get_ctrl(limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1328
        if( loop->is_member(get_loop(limit_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1329
          continue;             // Both inputs are loop varying; cannot RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1330
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1331
      // Here we know 'limit' is loop invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1332
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1333
      // 'limit' maybe pinned below the zero trip test (probably from a
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1334
      // previous round of rce), in which case, it can't be used in the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1335
      // zero trip test expression which must occur before the zero test's if.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1336
      if( limit_c == ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1337
        continue;  // Don't rce this check but continue looking for other candidates.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1338
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1339
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1340
      // Check for scaled induction variable plus an offset
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1341
      Node *offset = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1342
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1343
      if (!is_scaled_iv_plus_offset(rc_exp, trip_counter, &scale_con, &offset)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1344
        continue;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1345
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1346
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1347
      Node *offset_c = get_ctrl(offset);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1348
      if( loop->is_member( get_loop(offset_c) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1349
        continue;               // Offset is not really loop invariant
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1350
      // Here we know 'offset' is loop invariant.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1351
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1352
      // As above for the 'limit', the 'offset' maybe pinned below the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1353
      // zero trip test.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1354
      if( offset_c == ctrl ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1355
        continue; // Don't rce this check but continue looking for other candidates.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1356
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1357
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1358
      // At this point we have the expression as:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1359
      //   scale_con * trip_counter + offset :: limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1360
      // where scale_con, offset and limit are loop invariant.  Trip_counter
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1361
      // monotonically increases by stride_con, a constant.  Both (or either)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1362
      // stride_con and scale_con can be negative which will flip about the
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1363
      // sense of the test.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1364
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1365
      // Adjust pre and main loop limits to guard the correct iteration set
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1366
      if( cmp->Opcode() == Op_CmpU ) {// Unsigned compare is really 2 tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1367
        if( b_test._test == BoolTest::lt ) { // Range checks always use lt
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1368
          // The overflow limit: scale*I+offset < limit
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1369
          add_constraint( stride_con, scale_con, offset, limit, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1370
          // The underflow limit: 0 <= scale*I+offset.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1371
          // Some math yields: -scale*I-(offset+1) < 0
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1372
          Node *plus_one = new (C, 3) AddINode( offset, one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1373
          register_new_node( plus_one, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1374
          Node *neg_offset = new (C, 3) SubINode( zero, plus_one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1375
          register_new_node( neg_offset, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1376
          add_constraint( stride_con, -scale_con, neg_offset, zero, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1377
          if (!conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1378
            conditional_rc = !loop->dominates_backedge(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1379
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1380
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1381
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1382
          if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1383
            tty->print_cr("missed RCE opportunity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1384
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1385
          continue;             // In release mode, ignore it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1386
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1387
      } else {                  // Otherwise work on normal compares
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1388
        switch( b_test._test ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1389
        case BoolTest::ge:      // Convert X >= Y to -X <= -Y
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1390
          scale_con = -scale_con;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1391
          offset = new (C, 3) SubINode( zero, offset );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1392
          register_new_node( offset, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1393
          limit  = new (C, 3) SubINode( zero, limit  );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1394
          register_new_node( limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1395
          // Fall into LE case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1396
        case BoolTest::le:      // Convert X <= Y to X < Y+1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1397
          limit = new (C, 3) AddINode( limit, one );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1398
          register_new_node( limit, pre_ctrl );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1399
          // Fall into LT case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1400
        case BoolTest::lt:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1401
          add_constraint( stride_con, scale_con, offset, limit, pre_ctrl, &pre_limit, &main_limit );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1402
          if (!conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1403
            conditional_rc = !loop->dominates_backedge(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1404
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1405
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1406
        default:
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1407
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1408
          if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1409
            tty->print_cr("missed RCE opportunity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1410
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1411
          continue;             // Unhandled case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1412
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1413
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1414
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1415
      // Kill the eliminated test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1416
      C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1417
      Node *kill_con = _igvn.intcon( 1-flip );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1418
      set_ctrl(kill_con, C->root());
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1419
      _igvn.hash_delete(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1420
      iff->set_req(1, kill_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1421
      _igvn._worklist.push(iff);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1422
      // Find surviving projection
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1423
      assert(iff->is_If(), "");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1424
      ProjNode* dp = ((IfNode*)iff)->proj_out(1-flip);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1425
      // Find loads off the surviving projection; remove their control edge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1426
      for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1427
        Node* cd = dp->fast_out(i); // Control-dependent node
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1428
        if( cd->is_Load() ) {   // Loads can now float around in the loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1429
          _igvn.hash_delete(cd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1430
          // Allow the load to float around in the loop, or before it
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1431
          // but NOT before the pre-loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1432
          cd->set_req(0, ctrl);   // ctrl, not NULL
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1433
          _igvn._worklist.push(cd);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1434
          --i;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1435
          --imax;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1436
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1437
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1438
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1439
    } // End of is IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1440
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1441
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1442
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1443
  // Update loop limits
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1444
  if (conditional_rc) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1445
    pre_limit = (stride_con > 0) ? (Node*)new (C,3) MinINode(pre_limit, orig_limit)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1446
                                 : (Node*)new (C,3) MaxINode(pre_limit, orig_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1447
    register_new_node(pre_limit, pre_ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1448
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1449
  _igvn.hash_delete(pre_opaq);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1450
  pre_opaq->set_req(1, pre_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1451
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1452
  // Note:: we are making the main loop limit no longer precise;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1453
  // need to round up based on stride.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1454
  if( stride_con != 1 && stride_con != -1 ) { // Cutout for common case
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1455
    // "Standard" round-up logic:  ([main_limit-init+(y-1)]/y)*y+init
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1456
    // Hopefully, compiler will optimize for powers of 2.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1457
    Node *ctrl = get_ctrl(main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1458
    Node *stride = cl->stride();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1459
    Node *init = cl->init_trip();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1460
    Node *span = new (C, 3) SubINode(main_limit,init);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1461
    register_new_node(span,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1462
    Node *rndup = _igvn.intcon(stride_con + ((stride_con>0)?-1:1));
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1463
    Node *add = new (C, 3) AddINode(span,rndup);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1464
    register_new_node(add,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1465
    Node *div = new (C, 3) DivINode(0,add,stride);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1466
    register_new_node(div,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1467
    Node *mul = new (C, 3) MulINode(div,stride);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1468
    register_new_node(mul,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1469
    Node *newlim = new (C, 3) AddINode(mul,init);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1470
    register_new_node(newlim,ctrl);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1471
    main_limit = newlim;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1472
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1473
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1474
  Node *main_cle = cl->loopexit();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1475
  Node *main_bol = main_cle->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1476
  // Hacking loop bounds; need private copies of exit test
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1477
  if( main_bol->outcnt() > 1 ) {// BoolNode shared?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1478
    _igvn.hash_delete(main_cle);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1479
    main_bol = main_bol->clone();// Clone a private BoolNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1480
    register_new_node( main_bol, main_cle->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1481
    main_cle->set_req(1,main_bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1482
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1483
  Node *main_cmp = main_bol->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1484
  if( main_cmp->outcnt() > 1 ) { // CmpNode shared?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1485
    _igvn.hash_delete(main_bol);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1486
    main_cmp = main_cmp->clone();// Clone a private CmpNode
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1487
    register_new_node( main_cmp, main_cle->in(0) );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1488
    main_bol->set_req(1,main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1489
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1490
  // Hack the now-private loop bounds
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1491
  _igvn.hash_delete(main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1492
  main_cmp->set_req(2, main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1493
  _igvn._worklist.push(main_cmp);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1494
  // The OpaqueNode is unshared by design
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1495
  _igvn.hash_delete(opqzm);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1496
  assert( opqzm->outcnt() == 1, "cannot hack shared node" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1497
  opqzm->set_req(1,main_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1498
  _igvn._worklist.push(opqzm);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1499
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1500
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1501
//------------------------------DCE_loop_body----------------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1502
// Remove simplistic dead code from loop body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1503
void IdealLoopTree::DCE_loop_body() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1504
  for( uint i = 0; i < _body.size(); i++ )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1505
    if( _body.at(i)->outcnt() == 0 )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1506
      _body.map( i--, _body.pop() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1507
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1508
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1509
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1510
//------------------------------adjust_loop_exit_prob--------------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1511
// Look for loop-exit tests with the 50/50 (or worse) guesses from the parsing stage.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1512
// Replace with a 1-in-10 exit guess.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1513
void IdealLoopTree::adjust_loop_exit_prob( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1514
  Node *test = tail();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1515
  while( test != _head ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1516
    uint top = test->Opcode();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1517
    if( top == Op_IfTrue || top == Op_IfFalse ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1518
      int test_con = ((ProjNode*)test)->_con;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1519
      assert(top == (uint)(test_con? Op_IfTrue: Op_IfFalse), "sanity");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1520
      IfNode *iff = test->in(0)->as_If();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1521
      if( iff->outcnt() == 2 ) {        // Ignore dead tests
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1522
        Node *bol = iff->in(1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1523
        if( bol && bol->req() > 1 && bol->in(1) &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1524
            ((bol->in(1)->Opcode() == Op_StorePConditional ) ||
1500
bea9a90f3e8f 6462850: generate biased locking code in C2 ideal graph
kvn
parents: 1433
diff changeset
  1525
             (bol->in(1)->Opcode() == Op_StoreIConditional ) ||
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1526
             (bol->in(1)->Opcode() == Op_StoreLConditional ) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1527
             (bol->in(1)->Opcode() == Op_CompareAndSwapI ) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1528
             (bol->in(1)->Opcode() == Op_CompareAndSwapL ) ||
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 212
diff changeset
  1529
             (bol->in(1)->Opcode() == Op_CompareAndSwapP ) ||
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 212
diff changeset
  1530
             (bol->in(1)->Opcode() == Op_CompareAndSwapN )))
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1531
          return;               // Allocation loops RARELY take backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1532
        // Find the OTHER exit path from the IF
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1533
        Node* ex = iff->proj_out(1-test_con);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1534
        float p = iff->_prob;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1535
        if( !phase->is_member( this, ex ) && iff->_fcnt == COUNT_UNKNOWN ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1536
          if( top == Op_IfTrue ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1537
            if( p < (PROB_FAIR + PROB_UNLIKELY_MAG(3))) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1538
              iff->_prob = PROB_STATIC_FREQUENT;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1539
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1540
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1541
            if( p > (PROB_FAIR - PROB_UNLIKELY_MAG(3))) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1542
              iff->_prob = PROB_STATIC_INFREQUENT;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1543
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1544
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1545
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1546
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1547
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1548
    test = phase->idom(test);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1549
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1550
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1551
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1552
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1553
//------------------------------policy_do_remove_empty_loop--------------------
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1554
// Micro-benchmark spamming.  Policy is to always remove empty loops.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1555
// The 'DO' part is to replace the trip counter with the value it will
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1556
// have on the last iteration.  This will break the loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1557
bool IdealLoopTree::policy_do_remove_empty_loop( PhaseIdealLoop *phase ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1558
  // Minimum size must be empty loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1559
  if( _body.size() > 7/*number of nodes in an empty loop*/ ) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1560
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1561
  if( !_head->is_CountedLoop() ) return false;     // Dead loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1562
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1563
  if( !cl->loopexit() ) return false; // Malformed loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1564
  if( !phase->is_member(this,phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue)) ) )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1565
    return false;             // Infinite loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1566
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1567
  if( PrintOpto )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1568
    tty->print_cr("Removing empty loop");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1569
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1570
#ifdef ASSERT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1571
  // Ensure only one phi which is the iv.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1572
  Node* iv = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1573
  for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1574
    Node* n = cl->fast_out(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1575
    if (n->Opcode() == Op_Phi) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1576
      assert(iv == NULL, "Too many phis" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1577
      iv = n;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1578
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1579
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1580
  assert(iv == cl->phi(), "Wrong phi" );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1581
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1582
  // Replace the phi at loop head with the final value of the last
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1583
  // iteration.  Then the CountedLoopEnd will collapse (backedge never
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1584
  // taken) and all loop-invariant uses of the exit values will be correct.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1585
  Node *phi = cl->phi();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1586
  Node *final = new (phase->C, 3) SubINode( cl->limit(), cl->stride() );
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1587
  phase->register_new_node(final,cl->in(LoopNode::EntryControl));
5901
c046f8e9c52b 6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents: 5547
diff changeset
  1588
  phase->_igvn.replace_node(phi,final);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1589
  phase->C->set_major_progress();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1590
  return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1591
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1592
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1593
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1594
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1595
//------------------------------iteration_split_impl---------------------------
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1596
bool IdealLoopTree::iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1597
  // Check and remove empty loops (spam micro-benchmarks)
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1598
  if( policy_do_remove_empty_loop(phase) )
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1599
    return true;  // Here we removed an empty loop
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1600
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1601
  bool should_peel = policy_peeling(phase); // Should we peel?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1602
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1603
  bool should_unswitch = policy_unswitching(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1604
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1605
  // Non-counted loops may be peeled; exactly 1 iteration is peeled.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1606
  // This removes loop-invariant tests (usually null checks).
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1607
  if( !_head->is_CountedLoop() ) { // Non-counted loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1608
    if (PartialPeelLoop && phase->partial_peel(this, old_new)) {
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1609
      // Partial peel succeeded so terminate this round of loop opts
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1610
      return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1611
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1612
    if( should_peel ) {            // Should we peel?
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1613
#ifndef PRODUCT
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1614
      if (PrintOpto) tty->print_cr("should_peel");
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1615
#endif
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1616
      phase->do_peeling(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1617
    } else if( should_unswitch ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1618
      phase->do_unswitching(this, old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1619
    }
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1620
    return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1621
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1622
  CountedLoopNode *cl = _head->as_CountedLoop();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1623
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1624
  if( !cl->loopexit() ) return true; // Ignore various kinds of broken loops
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1625
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1626
  // Do nothing special to pre- and post- loops
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1627
  if( cl->is_pre_loop() || cl->is_post_loop() ) return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1628
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1629
  // Compute loop trip count from profile data
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1630
  compute_profile_trip_cnt(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1631
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1632
  // Before attempting fancy unrolling, RCE or alignment, see if we want
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1633
  // to completely unroll this loop or do loop unswitching.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1634
  if( cl->is_normal_loop() ) {
2876
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1635
    if (should_unswitch) {
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1636
      phase->do_unswitching(this, old_new);
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1637
      return true;
b8ebc85d0b47 6837146: Should perform unswitch before maximally unroll in loop transformation
cfang
parents: 2131
diff changeset
  1638
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1639
    bool should_maximally_unroll =  policy_maximally_unroll(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1640
    if( should_maximally_unroll ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1641
      // Here we did some unrolling and peeling.  Eventually we will
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1642
      // completely unroll this loop and it will no longer be a loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1643
      phase->do_maximally_unroll(this,old_new);
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1644
      return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1645
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1646
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1647
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1648
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1649
  // Counted loops may be peeled, may need some iterations run up
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1650
  // front for RCE, and may want to align loop refs to a cache
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1651
  // line.  Thus we clone a full loop up front whose trip count is
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1652
  // at least 1 (if peeling), but may be several more.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1653
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1654
  // The main loop will start cache-line aligned with at least 1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1655
  // iteration of the unrolled body (zero-trip test required) and
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1656
  // will have some range checks removed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1657
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1658
  // A post-loop will finish any odd iterations (leftover after
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1659
  // unrolling), plus any needed for RCE purposes.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1660
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1661
  bool should_unroll = policy_unroll(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1662
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1663
  bool should_rce = policy_range_check(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1664
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1665
  bool should_align = policy_align(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1666
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1667
  // If not RCE'ing (iteration splitting) or Aligning, then we do not
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1668
  // need a pre-loop.  We may still need to peel an initial iteration but
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1669
  // we will not be needing an unknown number of pre-iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1670
  //
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1671
  // Basically, if may_rce_align reports FALSE first time through,
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1672
  // we will not be able to later do RCE or Aligning on this loop.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1673
  bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1674
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1675
  // If we have any of these conditions (RCE, alignment, unrolling) met, then
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1676
  // we switch to the pre-/main-/post-loop model.  This model also covers
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1677
  // peeling.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1678
  if( should_rce || should_align || should_unroll ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1679
    if( cl->is_normal_loop() )  // Convert to 'pre/main/post' loops
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1680
      phase->insert_pre_post_loops(this,old_new, !may_rce_align);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1681
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1682
    // Adjust the pre- and main-loop limits to let the pre and post loops run
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1683
    // with full checks, but the main-loop with no checks.  Remove said
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1684
    // checks from the main body.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1685
    if( should_rce )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1686
      phase->do_range_check(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1687
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1688
    // Double loop body for unrolling.  Adjust the minimum-trip test (will do
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1689
    // twice as many iterations as before) and the main body limit (only do
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1690
    // an even number of trips).  If we are peeling, we might enable some RCE
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1691
    // and we'd rather unroll the post-RCE'd loop SO... do not unroll if
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1692
    // peeling.
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1693
      if( should_unroll && !should_peel )
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1694
        phase->do_unroll(this,old_new, true);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1695
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1696
    // Adjust the pre-loop limits to align the main body
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1697
    // iterations.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1698
    if( should_align )
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1699
      Unimplemented();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1700
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1701
  } else {                      // Else we have an unchanged counted loop
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1702
    if( should_peel )           // Might want to peel but do nothing else
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1703
      phase->do_peeling(this,old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1704
  }
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1705
  return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1706
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1707
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1708
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1709
//=============================================================================
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1710
//------------------------------iteration_split--------------------------------
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1711
bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1712
  // Recursively iteration split nested loops
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1713
  if( _child && !_child->iteration_split( phase, old_new ))
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1714
    return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1715
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1716
  // Clean out prior deadwood
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1717
  DCE_loop_body();
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1718
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1719
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1720
  // Look for loop-exit tests with my 50/50 guesses from the Parsing stage.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1721
  // Replace with a 1-in-10 exit guess.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1722
  if( _parent /*not the root loop*/ &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1723
      !_irreducible &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1724
      // Also ignore the occasional dead backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1725
      !tail()->is_top() ) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1726
    adjust_loop_exit_prob(phase);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1727
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1728
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1729
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1730
  // Gate unrolling, RCE and peeling efforts.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1731
  if( !_child &&                // If not an inner loop, do not split
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1732
      !_irreducible &&
212
cd4963e67949 6667612: (Escape Analysis) disable loop cloning if it has a scalar replaceable allocation
kvn
parents: 1
diff changeset
  1733
      _allow_optimizations &&
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1734
      !tail()->is_top() ) {     // Also ignore the occasional dead backedge
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1735
    if (!_has_call) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1736
        if (!iteration_split_impl( phase, old_new )) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1737
          return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1738
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1739
    } else if (policy_unswitching(phase)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1740
      phase->do_unswitching(this, old_new);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1741
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1742
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1743
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1744
  // Minor offset re-organization to remove loop-fallout uses of
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1745
  // trip counter.
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1746
  if( _head->is_CountedLoop() ) phase->reorg_offsets( this );
1433
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1747
  if( _next && !_next->iteration_split( phase, old_new ))
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1748
    return false;
ee60bd139c07 6743188: incomplete fix for 6700047 C2 failed in idom_no_update
never
parents: 1432
diff changeset
  1749
  return true;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1750
}
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1751
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1752
//-------------------------------is_uncommon_trap_proj----------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1753
// Return true if proj is the form of "proj->[region->..]call_uct"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1754
bool PhaseIdealLoop::is_uncommon_trap_proj(ProjNode* proj, bool must_reason_predicate) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1755
  int path_limit = 10;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1756
  assert(proj, "invalid argument");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1757
  Node* out = proj;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1758
  for (int ct = 0; ct < path_limit; ct++) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1759
    out = out->unique_ctrl_out();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1760
    if (out == NULL || out->is_Root() || out->is_Start())
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1761
      return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1762
    if (out->is_CallStaticJava()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1763
      int req = out->as_CallStaticJava()->uncommon_trap_request();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1764
      if (req != 0) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1765
        Deoptimization::DeoptReason reason = Deoptimization::trap_request_reason(req);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1766
        if (!must_reason_predicate || reason == Deoptimization::Reason_predicate){
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1767
           return true;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1768
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1769
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1770
      return false; // don't do further after call
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1771
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1772
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1773
  return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1774
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1775
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1776
//-------------------------------is_uncommon_trap_if_pattern-------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1777
// Return true  for "if(test)-> proj -> ...
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1778
//                          |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1779
//                          V
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1780
//                      other_proj->[region->..]call_uct"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1781
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1782
// "must_reason_predicate" means the uct reason must be Reason_predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1783
bool PhaseIdealLoop::is_uncommon_trap_if_pattern(ProjNode *proj, bool must_reason_predicate) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1784
  Node *in0 = proj->in(0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1785
  if (!in0->is_If()) return false;
5031
80fd89e1bd57 6930116: loop predication code does not handle If nodes with only one projection
kvn
parents: 5026
diff changeset
  1786
  // Variation of a dead If node.
80fd89e1bd57 6930116: loop predication code does not handle If nodes with only one projection
kvn
parents: 5026
diff changeset
  1787
  if (in0->outcnt() < 2)  return false;
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1788
  IfNode* iff = in0->as_If();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1789
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1790
  // we need "If(Conv2B(Opaque1(...)))" pattern for must_reason_predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1791
  if (must_reason_predicate) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1792
    if (iff->in(1)->Opcode() != Op_Conv2B ||
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1793
       iff->in(1)->in(1)->Opcode() != Op_Opaque1) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1794
      return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1795
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1796
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1797
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1798
  ProjNode* other_proj = iff->proj_out(1-proj->_con)->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1799
  return is_uncommon_trap_proj(other_proj, must_reason_predicate);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1800
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1801
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1802
//------------------------------create_new_if_for_predicate------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1803
// create a new if above the uct_if_pattern for the predicate to be promoted.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1804
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1805
//          before                                after
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1806
//        ----------                           ----------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1807
//           ctrl                                 ctrl
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1808
//            |                                     |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1809
//            |                                     |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1810
//            v                                     v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1811
//           iff                                 new_iff
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1812
//          /    \                                /      \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1813
//         /      \                              /        \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1814
//        v        v                            v          v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1815
//  uncommon_proj cont_proj                   if_uct     if_cont
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1816
// \      |        |                           |          |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1817
//  \     |        |                           |          |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1818
//   v    v        v                           |          v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1819
//     rgn       loop                          |         iff
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1820
//      |                                      |        /     \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1821
//      |                                      |       /       \
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1822
//      v                                      |      v         v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1823
// uncommon_trap                               | uncommon_proj cont_proj
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1824
//                                           \  \    |           |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1825
//                                            \  \   |           |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1826
//                                             v  v  v           v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1827
//                                               rgn           loop
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1828
//                                                |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1829
//                                                |
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1830
//                                                v
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1831
//                                           uncommon_trap
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1832
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1833
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1834
// We will create a region to guard the uct call if there is no one there.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1835
// The true projecttion (if_cont) of the new_iff is returned.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1836
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1837
  assert(is_uncommon_trap_if_pattern(cont_proj, true), "must be a uct if pattern!");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1838
  IfNode* iff = cont_proj->in(0)->as_If();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1839
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1840
  ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1841
  Node     *rgn   = uncommon_proj->unique_ctrl_out();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1842
  assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1843
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1844
  if (!rgn->is_Region()) { // create a region to guard the call
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1845
    assert(rgn->is_Call(), "must be call uct");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1846
    CallNode* call = rgn->as_Call();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1847
    rgn = new (C, 1) RegionNode(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1848
    _igvn.set_type(rgn, rgn->bottom_type());
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1849
    rgn->add_req(uncommon_proj);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1850
    set_idom(rgn, idom(uncommon_proj), dom_depth(uncommon_proj)+1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1851
    _igvn.hash_delete(call);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1852
    call->set_req(0, rgn);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1853
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1854
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1855
  // Create new_iff
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1856
  uint  iffdd  = dom_depth(iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1857
  IdealLoopTree* lp = get_loop(iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1858
  IfNode *new_iff = new (C, 2) IfNode(iff->in(0), NULL, iff->_prob, iff->_fcnt);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1859
  register_node(new_iff, lp, idom(iff), iffdd);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1860
  Node *if_cont = new (C, 1) IfTrueNode(new_iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1861
  Node *if_uct  = new (C, 1) IfFalseNode(new_iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1862
  if (cont_proj->is_IfFalse()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1863
    // Swap
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1864
    Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1865
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1866
  register_node(if_cont, lp, new_iff, iffdd);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1867
  register_node(if_uct, get_loop(rgn), new_iff, iffdd);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1868
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1869
  // if_cont to iff
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1870
  _igvn.hash_delete(iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1871
  iff->set_req(0, if_cont);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1872
  set_idom(iff, if_cont, dom_depth(iff));
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1873
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1874
  // if_uct to rgn
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1875
  _igvn.hash_delete(rgn);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1876
  rgn->add_req(if_uct);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1877
  Node* ridom = idom(rgn);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1878
  Node* nrdom = dom_lca(ridom, new_iff);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1879
  set_idom(rgn, nrdom, dom_depth(rgn));
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1880
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1881
  // rgn must have no phis
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1882
  assert(!rgn->as_Region()->has_phi(), "region must have no phis");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1883
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1884
  return if_cont->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1885
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1886
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1887
//------------------------------find_predicate_insertion_point--------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1888
// Find a good location to insert a predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1889
ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1890
  if (start_c == C->root() || !start_c->is_Proj())
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1891
    return NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1892
  if (is_uncommon_trap_if_pattern(start_c->as_Proj(), true/*Reason_Predicate*/)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1893
    return start_c->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1894
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1895
  return NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1896
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1897
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1898
//------------------------------Invariance-----------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1899
// Helper class for loop_predication_impl to compute invariance on the fly and
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1900
// clone invariants.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1901
class Invariance : public StackObj {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1902
  VectorSet _visited, _invariant;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1903
  Node_Stack _stack;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1904
  VectorSet _clone_visited;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1905
  Node_List _old_new; // map of old to new (clone)
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1906
  IdealLoopTree* _lpt;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1907
  PhaseIdealLoop* _phase;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1908
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1909
  // Helper function to set up the invariance for invariance computation
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1910
  // If n is a known invariant, set up directly. Otherwise, look up the
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1911
  // the possibility to push n onto the stack for further processing.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1912
  void visit(Node* use, Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1913
    if (_lpt->is_invariant(n)) { // known invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1914
      _invariant.set(n->_idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1915
    } else if (!n->is_CFG()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1916
      Node *n_ctrl = _phase->ctrl_or_self(n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1917
      Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1918
      if (_phase->is_dominator(n_ctrl, u_ctrl)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1919
        _stack.push(n, n->in(0) == NULL ? 1 : 0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1920
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1921
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1922
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1923
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1924
  // Compute invariance for "the_node" and (possibly) all its inputs recursively
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1925
  // on the fly
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1926
  void compute_invariance(Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1927
    assert(_visited.test(n->_idx), "must be");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1928
    visit(n, n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1929
    while (_stack.is_nonempty()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1930
      Node*  n = _stack.node();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1931
      uint idx = _stack.index();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1932
      if (idx == n->req()) { // all inputs are processed
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1933
        _stack.pop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1934
        // n is invariant if it's inputs are all invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1935
        bool all_inputs_invariant = true;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1936
        for (uint i = 0; i < n->req(); i++) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1937
          Node* in = n->in(i);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1938
          if (in == NULL) continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1939
          assert(_visited.test(in->_idx), "must have visited input");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1940
          if (!_invariant.test(in->_idx)) { // bad guy
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1941
            all_inputs_invariant = false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1942
            break;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1943
          }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1944
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1945
        if (all_inputs_invariant) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1946
          _invariant.set(n->_idx); // I am a invariant too
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1947
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1948
      } else { // process next input
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1949
        _stack.set_index(idx + 1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1950
        Node* m = n->in(idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1951
        if (m != NULL && !_visited.test_set(m->_idx)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1952
          visit(n, m);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1953
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1954
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1955
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1956
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1957
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1958
  // Helper function to set up _old_new map for clone_nodes.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1959
  // If n is a known invariant, set up directly ("clone" of n == n).
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1960
  // Otherwise, push n onto the stack for real cloning.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1961
  void clone_visit(Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1962
    assert(_invariant.test(n->_idx), "must be invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1963
    if (_lpt->is_invariant(n)) { // known invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1964
      _old_new.map(n->_idx, n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1965
    } else{ // to be cloned
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1966
      assert (!n->is_CFG(), "should not see CFG here");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1967
      _stack.push(n, n->in(0) == NULL ? 1 : 0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1968
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1969
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1970
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1971
  // Clone "n" and (possibly) all its inputs recursively
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1972
  void clone_nodes(Node* n, Node* ctrl) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1973
    clone_visit(n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1974
    while (_stack.is_nonempty()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1975
      Node*  n = _stack.node();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1976
      uint idx = _stack.index();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1977
      if (idx == n->req()) { // all inputs processed, clone n!
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1978
        _stack.pop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1979
        // clone invariant node
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1980
        Node* n_cl = n->clone();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1981
        _old_new.map(n->_idx, n_cl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1982
        _phase->register_new_node(n_cl, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1983
        for (uint i = 0; i < n->req(); i++) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1984
          Node* in = n_cl->in(i);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1985
          if (in == NULL) continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1986
          n_cl->set_req(i, _old_new[in->_idx]);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1987
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1988
      } else { // process next input
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1989
        _stack.set_index(idx + 1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1990
        Node* m = n->in(idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1991
        if (m != NULL && !_clone_visited.test_set(m->_idx)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1992
          clone_visit(m); // visit the input
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1993
        }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1994
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1995
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1996
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1997
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1998
 public:
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  1999
  Invariance(Arena* area, IdealLoopTree* lpt) :
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2000
    _lpt(lpt), _phase(lpt->_phase),
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2001
    _visited(area), _invariant(area), _stack(area, 10 /* guess */),
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2002
    _clone_visited(area), _old_new(area)
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2003
  {}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2004
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2005
  // Map old to n for invariance computation and clone
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2006
  void map_ctrl(Node* old, Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2007
    assert(old->is_CFG() && n->is_CFG(), "must be");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2008
    _old_new.map(old->_idx, n); // "clone" of old is n
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2009
    _invariant.set(old->_idx);  // old is invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2010
    _clone_visited.set(old->_idx);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2011
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2012
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2013
  // Driver function to compute invariance
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2014
  bool is_invariant(Node* n) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2015
    if (!_visited.test_set(n->_idx))
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2016
      compute_invariance(n);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2017
    return (_invariant.test(n->_idx) != 0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2018
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2019
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2020
  // Driver function to clone invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2021
  Node* clone(Node* n, Node* ctrl) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2022
    assert(ctrl->is_CFG(), "must be");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2023
    assert(_invariant.test(n->_idx), "must be an invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2024
    if (!_clone_visited.test(n->_idx))
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2025
      clone_nodes(n, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2026
    return _old_new[n->_idx];
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2027
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2028
};
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2029
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2030
//------------------------------is_range_check_if -----------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2031
// Returns true if the predicate of iff is in "scale*iv + offset u< load_range(ptr)" format
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2032
// Note: this function is particularly designed for loop predication. We require load_range
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2033
//       and offset to be loop invariant computed on the fly by "invar"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2034
bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2035
  if (!is_loop_exit(iff)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2036
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2037
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2038
  if (!iff->in(1)->is_Bool()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2039
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2040
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2041
  const BoolNode *bol = iff->in(1)->as_Bool();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2042
  if (bol->_test._test != BoolTest::lt) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2043
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2044
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2045
  if (!bol->in(1)->is_Cmp()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2046
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2047
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2048
  const CmpNode *cmp = bol->in(1)->as_Cmp();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2049
  if (cmp->Opcode() != Op_CmpU ) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2050
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2051
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2052
  if (cmp->in(2)->Opcode() != Op_LoadRange) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2053
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2054
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2055
  LoadRangeNode* lr = (LoadRangeNode*)cmp->in(2);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2056
  if (!invar.is_invariant(lr)) { // loadRange must be invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2057
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2058
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2059
  Node *iv     = _head->as_CountedLoop()->phi();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2060
  int   scale  = 0;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2061
  Node *offset = NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2062
  if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2063
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2064
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2065
  if(offset && !invar.is_invariant(offset)) { // offset must be invariant
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2066
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2067
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2068
  return true;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2069
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2070
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2071
//------------------------------rc_predicate-----------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2072
// Create a range check predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2073
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2074
// for (i = init; i < limit; i += stride) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2075
//    a[scale*i+offset]
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2076
// }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2077
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2078
// Compute max(scale*i + offset) for init <= i < limit and build the predicate
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2079
// as "max(scale*i + offset) u< a.length".
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2080
//
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2081
// There are two cases for max(scale*i + offset):
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2082
// (1) stride*scale > 0
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2083
//   max(scale*i + offset) = scale*(limit-stride) + offset
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2084
// (2) stride*scale < 0
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2085
//   max(scale*i + offset) = scale*init + offset
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2086
BoolNode* PhaseIdealLoop::rc_predicate(Node* ctrl,
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2087
                                       int scale, Node* offset,
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2088
                                       Node* init, Node* limit, Node* stride,
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2089
                                       Node* range, bool upper) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2090
  DEBUG_ONLY(ttyLocker ttyl);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2091
  if (TraceLoopPredicate) tty->print("rc_predicate ");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2092
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2093
  Node* max_idx_expr  = init;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2094
  int stride_con = stride->get_int();
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2095
  if ((stride_con > 0) == (scale > 0) == upper) {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2096
    max_idx_expr = new (C, 3) SubINode(limit, stride);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2097
    register_new_node(max_idx_expr, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2098
    if (TraceLoopPredicate) tty->print("(limit - stride) ");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2099
  } else {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2100
    if (TraceLoopPredicate) tty->print("init ");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2101
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2102
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2103
  if (scale != 1) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2104
    ConNode* con_scale = _igvn.intcon(scale);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2105
    max_idx_expr = new (C, 3) MulINode(max_idx_expr, con_scale);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2106
    register_new_node(max_idx_expr, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2107
    if (TraceLoopPredicate) tty->print("* %d ", scale);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2108
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2109
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2110
  if (offset && (!offset->is_Con() || offset->get_int() != 0)){
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2111
    max_idx_expr = new (C, 3) AddINode(max_idx_expr, offset);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2112
    register_new_node(max_idx_expr, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2113
    if (TraceLoopPredicate)
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2114
      if (offset->is_Con()) tty->print("+ %d ", offset->get_int());
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2115
      else tty->print("+ offset ");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2116
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2117
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2118
  CmpUNode* cmp = new (C, 3) CmpUNode(max_idx_expr, range);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2119
  register_new_node(cmp, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2120
  BoolNode* bol = new (C, 2) BoolNode(cmp, BoolTest::lt);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2121
  register_new_node(bol, ctrl);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2122
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2123
  if (TraceLoopPredicate) tty->print_cr("<u range");
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2124
  return bol;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2125
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2126
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2127
//------------------------------ loop_predication_impl--------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2128
// Insert loop predicates for null checks and range checks
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2129
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2130
  if (!UseLoopPredicate) return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2131
5026
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2132
  if (!loop->_head->is_Loop()) {
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2133
    // Could be a simple region when irreducible loops are present.
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2134
    return false;
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2135
  }
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2136
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2137
  CountedLoopNode *cl = NULL;
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2138
  if (loop->_head->is_CountedLoop()) {
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2139
    cl = loop->_head->as_CountedLoop();
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2140
    // do nothing for iteration-splitted loops
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2141
    if (!cl->is_normal_loop()) return false;
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2142
  }
51f0e7aa2245 6927049: assert(is_Loop(),"invalid node class")
never
parents: 4643
diff changeset
  2143
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2144
  // Too many traps seen?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2145
  bool tmt = C->too_many_traps(C->method(), 0, Deoptimization::Reason_predicate);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2146
  int tc = C->trap_count(Deoptimization::Reason_predicate);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2147
  if (tmt || tc > 0) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2148
    if (TraceLoopPredicate) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2149
      tty->print_cr("too many predicate traps: %d", tc);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2150
      C->method()->print(); // which method has too many predicate traps
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2151
      tty->print_cr("");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2152
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2153
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2154
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2155
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2156
  LoopNode *lpn  = loop->_head->as_Loop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2157
  Node* entry = lpn->in(LoopNode::EntryControl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2158
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2159
  ProjNode *predicate_proj = find_predicate_insertion_point(entry);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2160
  if (!predicate_proj){
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2161
#ifndef PRODUCT
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2162
    if (TraceLoopPredicate) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2163
      tty->print("missing predicate:");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2164
      loop->dump_head();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2165
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2166
#endif
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2167
    return false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2168
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2169
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2170
  ConNode* zero = _igvn.intcon(0);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2171
  set_ctrl(zero, C->root());
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2172
  Node *cond_false = new (C, 2) Conv2BNode(zero);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2173
  register_new_node(cond_false, C->root());
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2174
  ConNode* one = _igvn.intcon(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2175
  set_ctrl(one, C->root());
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2176
  Node *cond_true = new (C, 2) Conv2BNode(one);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2177
  register_new_node(cond_true, C->root());
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2178
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2179
  ResourceArea *area = Thread::current()->resource_area();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2180
  Invariance invar(area, loop);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2181
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2182
  // Create list of if-projs such that a newer proj dominates all older
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2183
  // projs in the list, and they all dominate loop->tail()
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2184
  Node_List if_proj_list(area);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2185
  LoopNode *head  = loop->_head->as_Loop();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2186
  Node *current_proj = loop->tail(); //start from tail
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2187
  while ( current_proj != head ) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2188
    if (loop == get_loop(current_proj) && // still in the loop ?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2189
        current_proj->is_Proj()        && // is a projection  ?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2190
        current_proj->in(0)->Opcode() == Op_If) { // is a if projection ?
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2191
      if_proj_list.push(current_proj);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2192
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2193
    current_proj = idom(current_proj);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2194
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2195
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2196
  bool hoisted = false; // true if at least one proj is promoted
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2197
  while (if_proj_list.size() > 0) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2198
    // Following are changed to nonnull when a predicate can be hoisted
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2199
    ProjNode* new_predicate_proj = NULL;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2200
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2201
    ProjNode* proj = if_proj_list.pop()->as_Proj();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2202
    IfNode*   iff  = proj->in(0)->as_If();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2203
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2204
    if (!is_uncommon_trap_if_pattern(proj)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2205
      if (loop->is_loop_exit(iff)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2206
        // stop processing the remaining projs in the list because the execution of them
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2207
        // depends on the condition of "iff" (iff->in(1)).
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2208
        break;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2209
      } else {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2210
        // Both arms are inside the loop. There are two cases:
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2211
        // (1) there is one backward branch. In this case, any remaining proj
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2212
        //     in the if_proj list post-dominates "iff". So, the condition of "iff"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2213
        //     does not determine the execution the remining projs directly, and we
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2214
        //     can safely continue.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2215
        // (2) both arms are forwarded, i.e. a diamond shape. In this case, "proj"
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2216
        //     does not dominate loop->tail(), so it can not be in the if_proj list.
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2217
        continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2218
      }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2219
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2220
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2221
    Node*     test = iff->in(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2222
    if (!test->is_Bool()){ //Conv2B, ...
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2223
      continue;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2224
    }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2225
    BoolNode* bol = test->as_Bool();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2226
    if (invar.is_invariant(bol)) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2227
      // Invariant test
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2228
      new_predicate_proj = create_new_if_for_predicate(predicate_proj);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2229
      Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2230
      BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2231
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2232
      // Negate test if necessary
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2233
      bool negated = false;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2234
      if (proj->_con != predicate_proj->_con) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2235
        new_predicate_bol = new (C, 2) BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate());
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2236
        register_new_node(new_predicate_bol, ctrl);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2237
        negated = true;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2238
      }
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2239
      IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2240
      _igvn.hash_delete(new_predicate_iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2241
      new_predicate_iff->set_req(1, new_predicate_bol);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2242
      if (TraceLoopPredicate) tty->print_cr("invariant if%s: %d", negated ? " negated" : "", new_predicate_iff->_idx);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2243
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2244
    } else if (cl != NULL && loop->is_range_check_if(iff, this, invar)) {
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2245
      assert(proj->_con == predicate_proj->_con, "must match");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2246
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2247
      // Range check for counted loops
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2248
      const Node*    cmp    = bol->in(1)->as_Cmp();
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2249
      Node*          idx    = cmp->in(1);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2250
      assert(!invar.is_invariant(idx), "index is variant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2251
      assert(cmp->in(2)->Opcode() == Op_LoadRange, "must be");
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2252
      Node* ld_rng = cmp->in(2); // LoadRangeNode
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2253
      assert(invar.is_invariant(ld_rng), "load range must be invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2254
      int scale    = 1;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2255
      Node* offset = zero;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2256
      bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2257
      assert(ok, "must be index expression");
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2258
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2259
      Node* init    = cl->init_trip();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2260
      Node* limit   = cl->limit();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2261
      Node* stride  = cl->stride();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2262
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2263
      // Build if's for the upper and lower bound tests.  The
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2264
      // lower_bound test will dominate the upper bound test and all
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2265
      // cloned or created nodes will use the lower bound test as
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2266
      // their declared control.
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2267
      ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2268
      ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2269
      assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2270
      Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2271
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2272
      // Perform cloning to keep Invariance state correct since the
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2273
      // late schedule will place invariant things in the loop.
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2274
      ld_rng = invar.clone(ld_rng, ctrl);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2275
      if (offset && offset != zero) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2276
        assert(invar.is_invariant(offset), "offset must be loop invariant");
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2277
        offset = invar.clone(offset, ctrl);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2278
      }
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2279
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2280
      // Test the lower bound
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2281
      Node*  lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, ld_rng, false);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2282
      IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2283
      _igvn.hash_delete(lower_bound_iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2284
      lower_bound_iff->set_req(1, lower_bound_bol);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2285
      if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2286
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2287
      // Test the upper bound
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2288
      Node* upper_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, ld_rng, true);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2289
      IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2290
      _igvn.hash_delete(upper_bound_iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2291
      upper_bound_iff->set_req(1, upper_bound_bol);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2292
      if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2293
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2294
      // Fall through into rest of the clean up code which will move
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2295
      // any dependent nodes onto the upper bound test.
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2296
      new_predicate_proj = upper_bound_proj;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2297
    } else {
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2298
      // The other proj of the "iff" is a uncommon trap projection, and we can assume
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2299
      // the other proj will not be executed ("executed" means uct raised).
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2300
      continue;
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2301
    }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2302
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2303
    // Success - attach condition (new_predicate_bol) to predicate if
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2304
    invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2305
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2306
    // Eliminate the old if in the loop body
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2307
    _igvn.hash_delete(iff);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2308
    iff->set_req(1, proj->is_IfFalse() ? cond_false : cond_true);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2309
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2310
    Node* ctrl = new_predicate_proj; // new control
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2311
    ProjNode* dp = proj;     // old control
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2312
    assert(get_loop(dp) == loop, "guaranteed at the time of collecting proj");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2313
    // Find nodes (depends only on the test) off the surviving projection;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2314
    // move them outside the loop with the control of proj_clone
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2315
    for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2316
      Node* cd = dp->fast_out(i); // Control-dependent node
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2317
      if (cd->depends_only_on_test()) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2318
        assert(cd->in(0) == dp, "");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2319
        _igvn.hash_delete(cd);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2320
        cd->set_req(0, ctrl); // ctrl, not NULL
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2321
        set_early_ctrl(cd);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2322
        _igvn._worklist.push(cd);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2323
        IdealLoopTree *new_loop = get_loop(get_ctrl(cd));
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2324
        if (new_loop != loop) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2325
          if (!loop->_child) loop->_body.yank(cd);
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2326
          if (!new_loop->_child ) new_loop->_body.push(cd);
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2327
        }
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2328
        --i;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2329
        --imax;
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2330
      }
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2331
    }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2332
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2333
    hoisted = true;
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2334
    C->set_major_progress();
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2335
  } // end while
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2336
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2337
#ifndef PRODUCT
5054
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2338
  // report that the loop predication has been actually performed
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2339
  // for this loop
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2340
  if (TraceLoopPredicate && hoisted) {
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2341
    tty->print("Loop Predication Performed:");
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2342
    loop->dump_head();
6d42dc4dd98c 6930043: C2: SIGSEGV in javasoft.sqe.tests.lang.arr017.arr01702.arr01702.loop_forw(II)I
never
parents: 5031
diff changeset
  2343
  }
4643
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2344
#endif
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2345
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2346
  return hoisted;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2347
}
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2348
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2349
//------------------------------loop_predication--------------------------------
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2350
// driver routine for loop predication optimization
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2351
bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2352
  bool hoisted = false;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2353
  // Recursively promote predicates
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2354
  if ( _child ) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2355
    hoisted = _child->loop_predication( phase);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2356
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2357
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2358
  // self
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2359
  if (!_irreducible && !tail()->is_top()) {
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2360
    hoisted |= phase->loop_predication_impl(this);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2361
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2362
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2363
  if ( _next ) { //sibling
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2364
    hoisted |= _next->loop_predication( phase);
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2365
  }
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2366
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2367
  return hoisted;
61c659c91c57 6894779: Loop Predication for Loop Optimizer in C2
cfang
parents: 3261
diff changeset
  2368
}